--- a
+++ b/notebooks/scraping.ipynb
@@ -0,0 +1,1027 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.support.ui import Select\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "from selenium.common.exceptions import TimeoutException\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "//*[@id=\"adv-check-status\"]/div/div[2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Radio button for 'Recruiting and not yet recruiting studies' is detected and visible: True\n"
+     ]
+    }
+   ],
+   "source": [
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "driver.get(\"https://clinicaltrials.gov\")\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the radio group container to be visible\n",
+    "    WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[2]/ctg-search-filters-form/div[2]'))\n",
+    "    )\n",
+    "\n",
+    "    # Option 1: Wait for the radio button using ID\n",
+    "    recruiting_radio_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div/div[2]'))\n",
+    "    )\n",
+    "\n",
+    "    # Option 2: Wait for the label and locate the radio button\n",
+    "    # recruiting_radio_label = WebDriverWait(driver, 20).until(\n",
+    "    #     EC.element_to_be_clickable((By.XPATH, \"//label[@for='adv-radio-status1']\"))\n",
+    "    # )\n",
+    "    # recruiting_radio_button = driver.find_element(By.XPATH, \"//input[@id='adv-radio-status1']\")\n",
+    "\n",
+    "    # Print if the radio button is displayed\n",
+    "    print(\"Radio button for 'Recruiting and not yet recruiting studies' is detected and visible:\", recruiting_radio_button.is_displayed())\n",
+    "    \n",
+    "   # Scroll the page to bring the element into view before clicking\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", recruiting_radio_button)\n",
+    "\n",
+    "    # Click the button after scrolling to it\n",
+    "    recruiting_radio_button.click()\n",
+    "\n",
+    "\n",
+    "except TimeoutException:\n",
+    "    print(\"The radio button was not found or is not interactable.\")\n",
+    "    print(driver.page_source)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Recruiting option selected successfully.\n",
+      "Search button clicked, navigating to the results page.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "from selenium.common.exceptions import TimeoutException\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver and open the page\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "driver.get(\"https://clinicaltrials.gov\")\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the radio group container to be visible\n",
+    "    WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[2]/ctg-search-filters-form/div[2]'))\n",
+    "    )\n",
+    "\n",
+    "    # Scroll down slowly to the radio button section\n",
+    "    driver.execute_script(\"window.scrollBy(0, 600);\")  # Adjust scrolling value as needed\n",
+    "    time.sleep(1)  # Give time for the scrolling action to be visually clear\n",
+    "\n",
+    "    # Locate and click the 'Recruiting and not yet recruiting studies' radio button\n",
+    "    recruiting_radio_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div/div[2]')) #//*[@id=\"adv-check-status\"]/div/div[2]\n",
+    "        # //*[@id=\"adv-check-status\"]/div/div[2]\n",
+    "    )\n",
+    "\n",
+    "    # Scroll the page to bring the radio button into view and click it\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", recruiting_radio_button)\n",
+    "    time.sleep(1)  # Pause before interacting\n",
+    "\n",
+    "    # Ensure the 'Recruiting and not yet recruiting' radio button is selected\n",
+    "    recruiting_radio_button.click()\n",
+    "    time.sleep(1)  # Wait a bit to confirm the selection\n",
+    "    \n",
+    "    # Optionally verify the selection\n",
+    "    selected_status = driver.find_element(By.XPATH, '//*[@id=\"adv-radio-status1\"]').is_selected()\n",
+    "    if selected_status:\n",
+    "        print(\"Recruiting option selected successfully.\")\n",
+    "    else:\n",
+    "        print(\"Failed to select Recruiting option.\")\n",
+    "\n",
+    "    # Now, wait for the search button to be visible\n",
+    "    search_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.element_to_be_clickable((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[3]/div/div/button'))\n",
+    "    )\n",
+    "\n",
+    "    # Scroll the page to bring the search button into view (just in case)\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", search_button)\n",
+    "    time.sleep(1)  # Wait a bit before clicking\n",
+    "\n",
+    "    # Click the search button\n",
+    "    search_button.click()\n",
+    "    print(\"Search button clicked, navigating to the results page.\")\n",
+    "\n",
+    "except TimeoutException:\n",
+    "    print(\"The search button was not found or is not interactable.\")\n",
+    "    print(driver.page_source)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Recruiting option selected successfully.\n",
+      "Search button clicked, navigating to the results page.\n",
+      "'Not yet recruiting' checkbox was already unselected.\n",
+      "An error occurred: Message: \n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver and open the main Clinical Trials page\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "driver.get(\"https://clinicaltrials.gov\")\n",
+    "\n",
+    "try:\n",
+    "    # ----- FIRST PAGE -----\n",
+    "    # Wait for the radio group container to be visible\n",
+    "    WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[2]/ctg-search-filters-form/div[2]'))\n",
+    "    )\n",
+    "\n",
+    "    # Scroll down slowly to the radio button section\n",
+    "    driver.execute_script(\"window.scrollBy(0, 600);\")  # Adjust scrolling value as needed\n",
+    "    time.sleep(1)  # Give time for the scrolling action to be visually clear\n",
+    "\n",
+    "\n",
+    "    # Locate and click the 'Recruiting and not yet recruiting studies' radio button\n",
+    "    recruiting_radio_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div/div[2]'))\n",
+    "    )\n",
+    "    \n",
+    "    # Scroll the page to bring the radio button into view and click it\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", recruiting_radio_button)\n",
+    "    time.sleep(1)  # Pause before interacting\n",
+    "    \n",
+    "    # Ensure the 'Recruiting and not yet recruiting' radio button is selected\n",
+    "    recruiting_radio_button.click()\n",
+    "    time.sleep(1)  # Wait a bit to confirm the selection\n",
+    "    \n",
+    "    # Optionally verify the selection\n",
+    "    selected_status = driver.find_element(By.XPATH, '//*[@id=\"adv-radio-status1\"]').is_selected()\n",
+    "    if selected_status:\n",
+    "        print(\"Recruiting option selected successfully.\")\n",
+    "    else:\n",
+    "        print(\"Failed to select Recruiting option.\")\n",
+    "        \n",
+    "    \n",
+    "    # Now, wait for the search button to be visible\n",
+    "    search_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.element_to_be_clickable((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[3]/div/div/button'))\n",
+    "    )\n",
+    "    \n",
+    "    # Scroll the page to bring the search button into view (just in case)\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", search_button)\n",
+    "    time.sleep(1)  # Wait a bit before clicking\n",
+    "    \n",
+    "    # Click the search button\n",
+    "    search_button.click()\n",
+    "    print(\"Search button clicked, navigating to the results page.\")\n",
+    "\n",
+    "   \n",
+    "    # Wait for the second page to load\n",
+    "    time.sleep(5)  # You may want to adjust this based on your internet speed\n",
+    "\n",
+    "    # ----- SECOND PAGE -----\n",
+    "    # Wait for the filter container to be visible on the second page\n",
+    "    filter_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-search-results-page/div[2]/section/div[1]/ctg-focus-your-search-panel/div/div[2]/ctg-search-filters-form'))\n",
+    "    )\n",
+    "\n",
+    "    # Scroll to the filter container\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", filter_container)\n",
+    "    time.sleep(1)\n",
+    "\n",
+    "    # Locate the \"Not yet recruiting\" checkbox and unselect it if it's selected\n",
+    "    not_yet_recruiting_checkbox = WebDriverWait(driver, 20).until(\n",
+    "        EC.element_to_be_clickable((By.XPATH, '//*[@id=\"adv-check-status\"]/div[2]/div[1]'))\n",
+    "    )\n",
+    "    \n",
+    "    # Unselect the checkbox if it's selected\n",
+    "    if not_yet_recruiting_checkbox.is_selected():\n",
+    "        not_yet_recruiting_checkbox.click()\n",
+    "        print(\"Unselected the 'Not yet recruiting' checkbox.\")\n",
+    "    else:\n",
+    "        print(\"'Not yet recruiting' checkbox was already unselected.\")\n",
+    "\n",
+    "    # Scroll to the Apply Filters button\n",
+    "    apply_filters_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.element_to_be_clickable((By.XPATH, '//*[@id=\"apply-filters\"]'))\n",
+    "    )\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", apply_filters_button)\n",
+    "    time.sleep(1)\n",
+    "\n",
+    "    # Click the Apply Filters button\n",
+    "    apply_filters_button.click()\n",
+    "    print(\"Clicked the 'Apply filters' button.\")\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Waiting for the filter container to load...\n",
+      "Filter container loaded successfully.\n",
+      "Verifying the 'Not yet recruiting' checkbox...\n",
+      "Checkbox 'Not yet recruiting' found.\n",
+      "Checkbox is already unselected.\n",
+      "Verifying the 'Apply Filters' button...\n",
+      "An error occurred: Message: \n",
+      "\n",
+      "Script completed. Keeping the browser window open.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver and open the specific page directly\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "driver.get(\"https://clinicaltrials.gov/search?aggFilters=status:not%20rec\")\n",
+    "\n",
+    "try:\n",
+    "    # ----- STEP 1: Verify filter container -----\n",
+    "    print(\"Waiting for the filter container to load...\")\n",
+    "    filter_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-search-results-page/div[2]/section/div[1]/ctg-focus-your-search-panel/div/div[2]/ctg-search-filters-form'))\n",
+    "    )\n",
+    "    print(\"Filter container loaded successfully.\")\n",
+    "    time.sleep(2)  # Pause for visual confirmation\n",
+    "\n",
+    "    # ----- STEP 2: Verify 'Not yet recruiting' checkbox -----\n",
+    "    print(\"Verifying the 'Not yet recruiting' checkbox...\")\n",
+    "    not_yet_recruiting_checkbox = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div[2]/div[1]'))\n",
+    "    )\n",
+    "    print(\"Checkbox 'Not yet recruiting' found.\")\n",
+    "    time.sleep(2)  # Pause for visual confirmation\n",
+    "\n",
+    "    # Check if it's selected, and unselect if it is\n",
+    "    if not_yet_recruiting_checkbox.is_selected():\n",
+    "        print(\"Checkbox is selected, unselecting it...\")\n",
+    "        not_yet_recruiting_checkbox.click()\n",
+    "        print(\"Checkbox unselected successfully.\")\n",
+    "    else:\n",
+    "        print(\"Checkbox is already unselected.\")\n",
+    "    time.sleep(2)  # Pause for visual confirmation\n",
+    "\n",
+    "    # ----- STEP 3: Verify 'Apply Filters' button -----\n",
+    "    print(\"Verifying the 'Apply Filters' button...\")\n",
+    "    apply_filters_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.element_to_be_clickable((By.XPATH, '//*[@id=\"apply-filters\"]'))\n",
+    "    )\n",
+    "    print(\"'Apply Filters' button is available.\")\n",
+    "    time.sleep(2)  # Pause for visual confirmation\n",
+    "\n",
+    "    # ----- STEP 4: Click 'Apply Filters' -----\n",
+    "    print(\"Clicking the 'Apply Filters' button...\")\n",
+    "    apply_filters_button.click()\n",
+    "    print(\"Clicked 'Apply Filters'.\")\n",
+    "    time.sleep(2)  # Pause for visual confirmation of the results after applying the filter\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Keep the browser open\n",
+    "    print(\"Script completed. Keeping the browser window open.\")\n",
+    "    input(\"Press Enter to close the browser window...\")\n",
+    "    driver.quit()  # Only quits when you press Enter\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Filter container loaded successfully.\n",
+      "Scrolling the filter container...\n",
+      "Script completed. Keeping the browser window open.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ----- FIRST PAGE -----\n",
+    "# Wait for the filter group container to be visible\n",
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "\n",
+    "# Navigate to the page\n",
+    "driver.get(\"https://clinicaltrials.gov/search?aggFilters=status:not%20rec\")\n",
+    "\n",
+    "try:\n",
+    "    # Step 1: Wait for the filter container to load\n",
+    "    filter_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-search-results-page/div[2]/section/div[1]/ctg-focus-your-search-panel/div/div[2]/ctg-search-filters-form'))\n",
+    "    )\n",
+    "    print(\"Filter container loaded successfully.\")\n",
+    "    time.sleep(2)\n",
+    "\n",
+    "    # Step 2: Scroll within the filter container using JavaScript\n",
+    "    print(\"Scrolling the filter container...\")\n",
+    "    \n",
+    "    # Scroll by a specific amount (e.g., 500 pixels)\n",
+    "    driver.execute_script(\"arguments[0].scrollTop = arguments[0].scrollTop + 500;\", filter_container)\n",
+    "    \n",
+    "    # Alternatively, you can scroll to the bottom of the container:\n",
+    "    # driver.execute_script(\"arguments[0].scrollTop = arguments[0].scrollHeight;\", filter_container)\n",
+    "    \n",
+    "    time.sleep(2)  # Pause for visual confirmation\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Keep the browser open\n",
+    "    print(\"Script completed. Keeping the browser window open.\")\n",
+    "    input(\"Press Enter to close the browser window...\")\n",
+    "    driver.quit()\n",
+    "  # Pause for visual confirmation\n",
+    "\n",
+    "\n",
+    "# # Locate and click the 'Recruiting and not yet recruiting studies' radio button\n",
+    "# recruiting_radio_button = WebDriverWait(driver, 20).until(\n",
+    "#     EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div/div[2]'))\n",
+    "# )\n",
+    "\n",
+    "# # Scroll the page to bring the radio button into view and click it\n",
+    "# driver.execute_script(\"arguments[0].scrollIntoView(true);\", recruiting_radio_button)\n",
+    "# time.sleep(1)  # Pause before interacting\n",
+    "\n",
+    "# # Ensure the 'Recruiting and not yet recruiting' radio button is selected\n",
+    "# recruiting_radio_button.click()\n",
+    "# time.sleep(1)  # Wait a bit to confirm the selection\n",
+    "\n",
+    "# # Optionally verify the selection\n",
+    "# selected_status = driver.find_element(By.XPATH, '//*[@id=\"adv-radio-status1\"]').is_selected()\n",
+    "# if selected_status:\n",
+    "#     print(\"Recruiting option selected successfully.\")\n",
+    "# else:\n",
+    "#     print(\"Failed to select Recruiting option.\")\n",
+    "    \n",
+    "\n",
+    "# # Now, wait for the search button to be visible\n",
+    "# search_button = WebDriverWait(driver, 20).until(\n",
+    "#     EC.element_to_be_clickable((By.XPATH, '//*[@id=\"main-content\"]/ctg-home/div/div[2]/ctg-home-search-panel/div/div[3]/div/div/button'))\n",
+    "# )\n",
+    "\n",
+    "# # Scroll the page to bring the search button into view (just in case)\n",
+    "# driver.execute_script(\"arguments[0].scrollIntoView(true);\", search_button)\n",
+    "# time.sleep(1)  # Wait a bit before clicking\n",
+    "\n",
+    "# # Click the search button\n",
+    "# search_button.click()\n",
+    "# print(\"Search button clicked, navigating to the results page.\")\n",
+    "\n",
+    "\n",
+    "# # Wait for the second page to load\n",
+    "# time.sleep(5)  # You may want to adjust this based on your internet speed\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Filter container loaded successfully.\n",
+      "Script completed. Keeping the browser window open.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "\n",
+    "# Navigate to the page\n",
+    "driver.get(\"https://clinicaltrials.gov/search?aggFilters=status:not%20rec\")\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the filter container to be visible\n",
+    "    filter_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"main-content\"]/ctg-search-results-page/div[2]/section/div[1]/ctg-focus-your-search-panel/div/div[2]/ctg-search-filters-form'))\n",
+    "    )\n",
+    "    \n",
+    "    print(\"Filter container loaded successfully.\")\n",
+    "    \n",
+    "    # Highlight the filter container by adding a red border around it\n",
+    "    driver.execute_script(\"arguments[0].style.border='3px solid red'\", filter_container)\n",
+    "    \n",
+    "    # Pause to visually inspect the browser\n",
+    "    time.sleep(5)  # Adjust this if needed for more time to inspect\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Keep the browser open for visual confirmation\n",
+    "    print(\"Script completed. Keeping the browser window open.\")\n",
+    "    input(\"Press Enter to close the browser window...\")\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Not Yet Recruiting button loaded successfully.\n",
+      "Script completed. Keeping the browser window open.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "\n",
+    "# Navigate to the page\n",
+    "driver.get(\"https://clinicaltrials.gov/search?aggFilters=status:not%20rec\")\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the \"Not Yet Recruiting\" radio button to be visible\n",
+    "    not_yet_recruiting_button = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div[2]/div[1]'))\n",
+    "    )\n",
+    "    \n",
+    "    print(\"Not Yet Recruiting button loaded successfully.\")\n",
+    "    \n",
+    "    # Highlight the \"Not Yet Recruiting\" button by adding a red border around it\n",
+    "    driver.execute_script(\"arguments[0].style.border='3px solid red'\", not_yet_recruiting_button)\n",
+    "    \n",
+    "    # Pause to visually inspect the browser\n",
+    "    time.sleep(5)  # Adjust this if needed for more time to inspect\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Keep the browser open for visual confirmation\n",
+    "    print(\"Script completed. Keeping the browser window open.\")\n",
+    "    input(\"Press Enter to close the browser window...\")\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Not Yet Recruiting checkbox loaded successfully.\n",
+      "Not Yet Recruiting checkbox is currently selected. Unselecting it...\n",
+      "Not Yet Recruiting checkbox has been unselected.\n",
+      "Script completed. Keeping the browser window open.\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "Interrupted by user",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[35], line 50\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     48\u001b[0m     \u001b[38;5;66;03m# Keep the browser open for visual confirmation\u001b[39;00m\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mScript completed. Keeping the browser window open.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 50\u001b[0m     \u001b[38;5;28;43minput\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPress Enter to close the browser window...\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     51\u001b[0m     driver\u001b[38;5;241m.\u001b[39mquit()\n",
+      "File \u001b[0;32m~/Desktop/patient-trials-matching/env/lib/python3.9/site-packages/ipykernel/kernelbase.py:1282\u001b[0m, in \u001b[0;36mKernel.raw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m   1280\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw_input was called, but this frontend does not support input requests.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1281\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m StdinNotImplementedError(msg)\n\u001b[0;32m-> 1282\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_input_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1283\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1284\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parent_ident\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshell\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1285\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_parent\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshell\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1286\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpassword\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1287\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Desktop/patient-trials-matching/env/lib/python3.9/site-packages/ipykernel/kernelbase.py:1325\u001b[0m, in \u001b[0;36mKernel._input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m   1322\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m   1323\u001b[0m     \u001b[38;5;66;03m# re-raise KeyboardInterrupt, to truncate traceback\u001b[39;00m\n\u001b[1;32m   1324\u001b[0m     msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInterrupted by user\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1325\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m(msg) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1326\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m   1327\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid Message:\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the driver\n",
+    "driver = webdriver.Chrome()\n",
+    "driver.set_window_size(1120, 1000)\n",
+    "\n",
+    "# Navigate to the page\n",
+    "driver.get(\"https://clinicaltrials.gov/search?aggFilters=status:not%20rec\")\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the \"Not Yet Recruiting\" checkbox to be visible\n",
+    "    not_yet_recruiting_checkbox = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"adv-check-status\"]/div[2]/div[1]'))\n",
+    "    )\n",
+    "    \n",
+    "    print(\"Not Yet Recruiting checkbox loaded successfully.\")\n",
+    "    \n",
+    "    # Check if the checkbox is selected\n",
+    "    checkbox_input = not_yet_recruiting_checkbox.find_element(By.TAG_NAME, 'input')\n",
+    "    \n",
+    "    if checkbox_input.is_selected():\n",
+    "        print(\"Not Yet Recruiting checkbox is currently selected. Unselecting it...\")\n",
+    "        \n",
+    "        # Scroll into view\n",
+    "        driver.execute_script(\"arguments[0].scrollIntoView(true);\", not_yet_recruiting_checkbox)\n",
+    "        \n",
+    "        # Use JavaScript to click the checkbox to unselect it\n",
+    "        driver.execute_script(\"arguments[0].click();\", checkbox_input)\n",
+    "\n",
+    "        # Highlight it after unselecting\n",
+    "        driver.execute_script(\"arguments[0].style.border='3px solid red'\", not_yet_recruiting_checkbox)\n",
+    "\n",
+    "        print(\"Not Yet Recruiting checkbox has been unselected.\")\n",
+    "    else:\n",
+    "        print(\"Not Yet Recruiting checkbox is already unselected.\")\n",
+    "\n",
+    "    # Pause to visually inspect the browser\n",
+    "    time.sleep(5)  # Adjust this if needed for more time to inspect\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Keep the browser open for visual confirmation\n",
+    "    print(\"Script completed. Keeping the browser window open.\")\n",
+    "    input(\"Press Enter to close the browser window...\")\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Eligibility Criteria container loaded successfully.\n",
+      "Eligibility Criteria:\n",
+      "Description\n",
+      "Inclusion Criteria:\n",
+      "Age ≥ 18 years at inclusion date\n",
+      "Schirmer's test ≤ 5 mm for both eyes\n",
+      "NIBUT ≤ 10 s for both eyes\n",
+      "Previous positive result of blood test for sample anti-Ro (SS-A) or anti-La (SS-B), as indicated by medical record or lab result shown by the subject.\n",
+      "The study subject reports having understood and have signed the Informed Consent Form (ICF) and is willing to comply with all investigation visits and assessments.\n",
+      "Women of childbearing potential must agree to use a reliable, medically approved form of contraception during the study participation until end of study.\n",
+      "Anticipated compliance with prescribed treatment and follow-up.\n",
+      "Exclusion Criteria:\n",
+      "Recently (12 months prior enrolment) undergone nasal, sinus, or ocular surgery.\n",
+      "Presence of an ocular or respiratory condition that could affect the study parameters such as active ocular infection/inflammation, glaucoma, diabetic retinopathy, or upper respiratory tract infection per the Investigator's judgement.\n",
+      "The study subject has a cognitive incapacity or language barrier precluding adequate understanding or cooperation.\n",
+      "Any severe diseases interfering with the performance, evaluation, and outcome of the clinical evaluation.\n",
+      "The study subject is considered by the Investigator to be unsuitable to participate in the investigation for any other reason.\n",
+      "Previous (within 30 days prior to enrolment) and concurrent treatment with another investigational drug/s or device/s.\n",
+      "Subject is pregnant or lactating or planning to get pregnant during the duration of the study.\n",
+      "Show less\n",
+      "Ages Eligible for Study\n",
+      "18 Years and older (Adult,  Older Adult )\n",
+      "Sexes Eligible for Study\n",
+      "All\n",
+      "Accepts Healthy Volunteers\n",
+      "No\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the Selenium WebDriver\n",
+    "driver = webdriver.Chrome()\n",
+    "\n",
+    "# Navigate to the webpage\n",
+    "driver.get('https://clinicaltrials.gov/study/NCT06626477#participation-criteria')\n",
+    "\n",
+    "# Wait for the \"Eligibility Criteria\" container to load\n",
+    "try:\n",
+    "    eligibility_criteria_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]'))\n",
+    "    )\n",
+    "    \n",
+    "    print(\"Eligibility Criteria container loaded successfully.\")\n",
+    "    \n",
+    "    # Extract the text from the eligibility container\n",
+    "    eligibility_text = eligibility_criteria_container.text\n",
+    "    print(\"Eligibility Criteria:\")\n",
+    "    print(eligibility_text)\n",
+    "    \n",
+    "    # Optionally save the data to a text file\n",
+    "    with open('eligibility_criteria.txt', 'w') as file:\n",
+    "        file.write(eligibility_text)\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Pause to visually inspect the browser\n",
+    "    time.sleep(5)\n",
+    "\n",
+    "    # Close the browser\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Eligibility Criteria container loaded successfully.\n",
+      "[]\n",
+      "[]\n",
+      "\n",
+      "Data successfully written to eligibility_criteria_dynamic.txt\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Set up the Selenium WebDriver\n",
+    "driver = webdriver.Chrome()\n",
+    "\n",
+    "# Navigate to the webpage\n",
+    "driver.get('https://clinicaltrials.gov/study/NCT06626386?aggFilters=status:rec&rank=2#participation-criteria')\n",
+    "\n",
+    "# Wait for the Eligibility Criteria container to load\n",
+    "try:\n",
+    "    eligibility_container = WebDriverWait(driver, 20).until(\n",
+    "        EC.visibility_of_element_located((By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]'))\n",
+    "    )\n",
+    "    \n",
+    "    print(\"Eligibility Criteria container loaded successfully.\")\n",
+    "    \n",
+    "    # Find and extract the headers (like \"Description\", \"Ages Eligible for Study\", etc.)\n",
+    "    headers = driver.find_elements(By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]//dt')\n",
+    "    \n",
+    "    # Find and extract the content corresponding to each header\n",
+    "    contents = driver.find_elements(By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]//dd')\n",
+    "    \n",
+    "    print(headers)\n",
+    "    print(contents)\n",
+    "    \n",
+    "    # Create a dictionary to store criteria dynamically\n",
+    "    criteria_data = {}\n",
+    "    \n",
+    "    # Loop over the headers and their respective contents\n",
+    "    for header, content in zip(headers, contents):\n",
+    "        header_text = header.text.strip()  # Get the header text\n",
+    "        content_text = content.text.strip()  # Get the corresponding content text\n",
+    "        \n",
+    "        # Store in dictionary\n",
+    "        criteria_data[header_text] = content_text\n",
+    "\n",
+    "    # Dynamically generate formatted criteria text\n",
+    "    formatted_criteria = \"\"\n",
+    "    for header, content in criteria_data.items():\n",
+    "        formatted_criteria += f\"{header}: {content}\\n\\n\"  # Add each header-content pair to the formatted string\n",
+    "\n",
+    "    # Print the formatted output to verify\n",
+    "    print(formatted_criteria)\n",
+    "    \n",
+    "    # Save the output to a text file\n",
+    "    with open('eligibility_criteria_dynamic.txt', 'w') as file:\n",
+    "        file.write(formatted_criteria)\n",
+    "    \n",
+    "    print(\"Data successfully written to eligibility_criteria_dynamic.txt\")\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(\"An error occurred:\", e)\n",
+    "\n",
+    "finally:\n",
+    "    # Pause to visually inspect the browser\n",
+    "    time.sleep(5)\n",
+    "\n",
+    "    # Close the browser\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Inclusion/Exclusion Criteria container loaded successfully.\n",
+      "Inclusion/Exclusion Criteria:\n",
+      "Description\n",
+      "Inclusion Criteria:\n",
+      "Men aged from 45 to 70 years\n",
+      "Informed consent for PSA measurement for prostate cancer screening\n",
+      "Exclusion Criteria:\n",
+      "Patients unable to give consent\n",
+      "Show less\n",
+      "Study Population\n",
+      "Male patients between the ages of 40 and 70 who receive a PSA test as part of their routine examination\n",
+      "Other Criteria container loaded successfully.\n",
+      "Other Criteria:\n",
+      "Ages Eligible for Study\n",
+      "45 Years to 70 Years (Adult,  Older Adult )\n",
+      "Sexes Eligible for Study\n",
+      "Male\n",
+      "Accepts Healthy Volunteers\n",
+      "No\n",
+      "Sampling Method\n",
+      "Non-Probability Sample\n",
+      "Data successfully written to clinical_trial_criteria.txt\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Initialize the WebDriver (Make sure to specify the correct path to your chromedriver)\n",
+    "driver = webdriver.Chrome()\n",
+    "\n",
+    "# Open the clinical trials page\n",
+    "driver.get(\"https://clinicaltrials.gov/study/NCT06626386?aggFilters=status:rec&rank=2#participation-criteria\")\n",
+    "\n",
+    "# Wait for the page to load completely\n",
+    "time.sleep(5)\n",
+    "\n",
+    "try:\n",
+    "    # Wait for the inclusion/exclusion criteria container to load\n",
+    "    inclusion_exclusion_criteria = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]/div[1]'))\n",
+    "    )\n",
+    "    print(\"Inclusion/Exclusion Criteria container loaded successfully.\")\n",
+    "\n",
+    "    # Scraping inclusion/exclusion criteria\n",
+    "    inclusion_exclusion_text = inclusion_exclusion_criteria.text\n",
+    "    print(\"Inclusion/Exclusion Criteria:\")\n",
+    "    print(inclusion_exclusion_text)\n",
+    "\n",
+    "    # Wait for the other criteria container to load\n",
+    "    other_criteria = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]/div[2]'))\n",
+    "    )\n",
+    "    print(\"Other Criteria container loaded successfully.\")\n",
+    "\n",
+    "    # Scraping other criteria\n",
+    "    other_criteria_text = other_criteria.text\n",
+    "    print(\"Other Criteria:\")\n",
+    "    print(other_criteria_text)\n",
+    "\n",
+    "    # Writing the scraped data to a text file\n",
+    "    with open('clinical_trial_criteria.txt', 'w') as file:\n",
+    "        file.write(\"Inclusion/Exclusion Criteria:\\n\")\n",
+    "        file.write(inclusion_exclusion_text + \"\\n\\n\")\n",
+    "        file.write(\"Other Criteria:\\n\")\n",
+    "        file.write(other_criteria_text)\n",
+    "\n",
+    "    print(\"Data successfully written to clinical_trial_criteria.txt\")\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"An error occurred: {str(e)}\")\n",
+    "\n",
+    "finally:\n",
+    "    # Optionally, keep the browser open to see the result\n",
+    "    time.sleep(10)  # Adjust the sleep time as needed\n",
+    "\n",
+    "    # Close the browser\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Inclusion Criteria element is now in view.\n",
+      "Inclusion Criteria:\n",
+      "Men aged from 45 to 70 years\n",
+      "Informed consent for PSA measurement for prostate cancer screening\n",
+      "Exclusion Criteria element is now in view.\n",
+      "Exclusion Criteria:\n",
+      "Patients unable to give consent\n",
+      "Other Criteria container loaded successfully.\n",
+      "Other Criteria:\n",
+      "Ages Eligible for Study\n",
+      "45 Years to 70 Years (Adult,  Older Adult )\n",
+      "Sexes Eligible for Study\n",
+      "Male\n",
+      "Accepts Healthy Volunteers\n",
+      "No\n",
+      "Sampling Method\n",
+      "Non-Probability Sample\n",
+      "Data successfully written to clinical_trial_criteria.txt\n"
+     ]
+    }
+   ],
+   "source": [
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.support.ui import WebDriverWait\n",
+    "from selenium.webdriver.support import expected_conditions as EC\n",
+    "import time\n",
+    "\n",
+    "# Initialize the WebDriver (Make sure to specify the correct path to your chromedriver)\n",
+    "driver = webdriver.Chrome()\n",
+    "\n",
+    "# Open the clinical trials page\n",
+    "driver.get(\"https://clinicaltrials.gov/study/NCT06626386?aggFilters=status:rec&rank=2#participation-criteria\")\n",
+    "\n",
+    "# Wait for the page to load completely\n",
+    "time.sleep(5)\n",
+    "\n",
+    "try:\n",
+    "    # Scroll the page to bring inclusion criteria into view\n",
+    "    inclusion_element = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"eligibility-criteria-description\"]/div/div/ul[1]')) #//*[@id=\"eligibility-criteria-description\"]/div/div/ul[1]\n",
+    "    )\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", inclusion_element)\n",
+    "    print(\"Inclusion Criteria element is now in view.\")\n",
+    "\n",
+    "    # Scraping inclusion criteria\n",
+    "    inclusion_criteria_text = inclusion_element.text\n",
+    "    print(\"Inclusion Criteria:\")\n",
+    "    print(inclusion_criteria_text)\n",
+    "\n",
+    "    # Scroll to exclusion criteria\n",
+    "    exclusion_element = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"eligibility-criteria-description\"]/div/div/ul[2]'))\n",
+    "    )\n",
+    "    driver.execute_script(\"arguments[0].scrollIntoView(true);\", exclusion_element)\n",
+    "    print(\"Exclusion Criteria element is now in view.\")\n",
+    "\n",
+    "    # Scraping exclusion criteria\n",
+    "    exclusion_criteria_text = exclusion_element.text\n",
+    "    print(\"Exclusion Criteria:\")\n",
+    "    print(exclusion_criteria_text)\n",
+    "    \n",
+    "    other_criteria = WebDriverWait(driver, 20).until(\n",
+    "        EC.presence_of_element_located((By.XPATH, '//*[@id=\"participation-criteria\"]/ctg-participation-criteria/div[2]/div/div[2]/div[2]'))\n",
+    "    )\n",
+    "    print(\"Other Criteria container loaded successfully.\")\n",
+    "    \n",
+    "    other_criteria_text = other_criteria.text\n",
+    "    print(\"Other Criteria:\")\n",
+    "    print(other_criteria_text)\n",
+    "\n",
+    "    # Writing the scraped data to a text file\n",
+    "    with open('clinical_trial_criteria.txt', 'w') as file:\n",
+    "        file.write(\"Inclusion Criteria:\\n\")\n",
+    "        file.write(inclusion_criteria_text + \"\\n\\n\")\n",
+    "        file.write(\"Exclusion Criteria:\\n\")\n",
+    "        file.write(exclusion_criteria_text + \"\\n\\n\")\n",
+    "        file.write(\"Other Criteria:\\n\")\n",
+    "        file.write(other_criteria_text)\n",
+    "\n",
+    "    print(\"Data successfully written to clinical_trial_criteria.txt\")\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"An error occurred: {str(e)}\")\n",
+    "\n",
+    "finally:\n",
+    "    # Optionally, keep the browser open to see the result\n",
+    "    time.sleep(5)  # Adjust the sleep time as needed\n",
+    "\n",
+    "    # Close the browser\n",
+    "    driver.quit()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Scraping Environment",
+   "language": "python",
+   "name": "scraping_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}