I am currently trying to scrap the data from a website CCMT 2021 OR/CR which has dynamic structure and table is present in it and to navigate to other page there is option of ‘Next’ or clicking on the ‘2’ or ‘3’ or required button. I want to extract all the data and keep it in a excel file. Previously I have extracted the similar link of CCMT 2023 OR/CR with same approach, and scraping of one page(21 rows) was taking approx 23 seconds and there was total of 383 pages and total time taken was aprrox 2 hours, and this CCMT 2021 OR/CR has also almost similar no. of rows in all pages. How can I scrap the table efficiently. I am beginner at Web Scraping and with the help of Chatgpt I was successful to scrap the previous website data.

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time

# Specify the path to chromedriver if not in PATH
chrome_driver_path = 'chromedriver.exe'  # Adjust the path as needed

# Initialize Chrome WebDriver
service = Service(chrome_driver_path)
driver = webdriver.Chrome(service=service)

# Function to scrape data from all pages
def scrape_all_pages(base_url):
    driver.get(base_url)  # Open the base URL

    # Initial wait to allow the page to load
    time.sleep(60)  # Increase this if the page loads very slowly

    # Container to store all the data
    all_data = []

    while True:
        try:
            # Wait until the table is present
            WebDriverWait(driver, 90).until(
                EC.presence_of_element_located((By.XPATH, '//*[@id="mainContent"]/app-jac-delhi/section/form/div/div[5]/div[3]/div/table/tbody/tr'))
            )

            # Locate the table rows on the current page
            data_elements = driver.find_elements(By.XPATH, '//*[@id="mainContent"]/app-jac-delhi/section/form/div/div[5]/div[3]/div/table/tbody/tr')

            # Extract data from each row (skip the header row)
            for row in data_elements:
                cols = row.find_elements(By.TAG_NAME, 'td')
                row_data = [col.text for col in cols]
                all_data.append(row_data)

            print(f"Scraped {len(data_elements)} rows from current page.")

            # Check if there is a "Next" button to go to the next page
            try:
                next_button = driver.find_element(By.XPATH,'//*[@id="mainContent"]/app-jac-delhi/section/form/div/div[5]/div[4]/pagination-controls/pagination-template/ul/li[10]')
                if "disabled" not in next_button.get_attribute("class"):
                    next_button.click()
                    # Wait for the next page to load
                    time.sleep(20)  # Increase this if the page loads very slowly
                else:
                    break  # Break the loop if the next page button is disabled
            except Exception as e:
                print(f"Next button not found or other exception: {e}")
                break  # Break the loop if the next button is not found
        except TimeoutException as e:
            print(f"Timeout waiting for table to load: {e}")
            break  # Break the loop if waiting for the table times out

    driver.quit()  # Close the browser
    return all_data

# Base URL of the webpage
base_url = "https://admissions.nic.in/admiss/admissions/orcrjacd/105012121"

# Scrape data from all pages
print(f"Scraping data from {base_url}...")
all_data = scrape_all_pages(base_url)

# Check if any data was scraped
if all_data:
    # Convert the scraped data to a DataFrame
    df = pd.DataFrame(all_data, columns=['SNo.', 'Round', 'Institute', 'PG Program', 'Group', 'Category', 'Max GATE Score', 'Min GATE Score'])

    # Save DataFrame to an Excel file
    output_file = 'scraped_data_2021.xlsx'
    df.to_excel(output_file, index=False)
    print(f"Data has been successfully scraped and saved to {output_file}.")
else:
    print("No data was scraped.")

I have heard of some method like multiprocessing, multithreading, Asynchronous Scraping, but I don’t know how to implement that.

Question: How can i make the Scraping fast and efficient?

Attempts:

1. I try to use this code:

# Configure Chrome options
chrome_options = Options()
chrome_options.add_argument("--headless")  # Run Chrome in headless mode
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")

# Initialize Chrome WebDriver
service = Service(chrome_driver_path)
driver = webdriver.Chrome(service=service, options=chrome_options)

but it was not helpful , the time taken was same.

2. I tried to decrease ‘TImeSleep time’ and ‘Web Driver Wait time’, time.sleep(10), WebDriverWait(driver, 10), but the the process speed was same i.e approx 23 seconds for 21 rows in a page.

Any suggestions or improvements to make this scraping process faster and more reliable would be greatly appreciated. Thank you!