I am trying web scrapping a website for Test data. I am stuck at extracting data from all pages, I chcecked the pagination code from source file, but my code is still returing page one data only. Can anyone help me what I am missing in my code.
Code I am using:
import csv
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def navigate_to_next_page():
try:
next_button = WebDriverWait(driver, 60).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, 'li.pagination-next a'))
)
next_button.click()
return True
except:
return False
def extract_test_data():
# Find all test divs
test_divs = WebDriverWait(driver, 60).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div.product"))
)
# Iterate over each test div to extract test name, URL, and price
for test_div in test_divs:
test_link = test_div.find_element(By.CSS_SELECTOR, "a.text-theme-colored")
test_name = test_link.text.strip()
test_url = test_link.get_attribute("href") # Extract href attribute for URL
test_price = test_div.find_element(By.CSS_SELECTOR, "span.amount").text.strip()
# Append test data to the list
all_test_data.append([test_url, test_name, test_price])
base_url = "https://www.tenetdiagnostics.in/book/tests?type=p"
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options=chrome_options)
driver.get(base_url)
all_test_data = []
while True:
extract_test_data()
if not navigate_to_next_page():
break
csv_file = "tenet_test_data.csv"
with open(csv_file, "w", newline="", encoding="utf-8") as file:
writer = csv.writer(file)
writer.writerow(["Test URL", "Test Name", "Test Price"]) # Write header
writer.writerows(all_test_data)
print("Test data saved to", csv_file)
driver.quit()
This code is giving me desired results but only from first page. I want to extract data from all pages.
Thanks In Advance!
New contributor
Diksha Ingle is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.