I want to extract the table as attached form this link: https://www.rfi.it/en/stations.html.
But I can extract the data from page 1. I need to extract the data of all pages.
Can you guys help me? Thanks

I’m not familiar with coding so I used github copilot, but it could not help me. Here it code:

from selenium import webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from bs4 import BeautifulSoup
import time

# Initialize WebDriver with the correct path to your Edge WebDriver executable
service = Service('D:/Downloads/edgedriver_win64/msedgedriver.exe')
driver = webdriver.Edge(service=service)

# Navigate to the URL
url = 'https://www.rfi.it/it/stazioni.html'
driver.get(url)

# Accept cookies if the popup appears
try:
    accept_cookies_button = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Accetta tutti i cookie")]'))
    )
    accept_cookies_button.click()
except Exception as e:
    print(f"No cookies button found or unable to click it. Error: {e}")

# Wait for the LISTA tab to be clickable and click it using JavaScript
lista_tab = WebDriverWait(driver, 10).until(
    EC.element_to_be_clickable((By.XPATH, '//a[contains(text(), "LISTA")]'))
)
driver.execute_script("arguments[0].click();", lista_tab)

# Initialize variables
data = []
max_pages = 5  # Set the maximum number of pages to scrape
current_page = 1

while current_page <= max_pages:
    print(f"Processing page {current_page}")
    
    # Parse the HTML content using BeautifulSoup
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    
    # Find the table element
    table = soup.find('table', class_='table table-striped table-hover')
    
    # Check if the table exists
    if table:
        headers = [header.text.strip() for header in table.find_all('th')]
        print(f"Headers: {headers}")  # Debugging statement
        if current_page == 1:
            data.append(headers)  # Add headers only once
        
        rows = table.find_all('tr')
        for row in rows:
            cols = row.find_all('td')
            if cols:  # Ensure the row has columns
                cols = [ele.text.strip() for ele in cols]
                print(f"Row data: {cols}")  # Debugging statement
                data.append(cols)
    else:
        print(f"No table found on page {current_page}. Stopping.")
        break
    
    # Find the "successivi" button and click it
    try:
        next_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, '//button[contains(@class, "pagination--next-btn")]'))
        )
        driver.execute_script("arguments[0].click();", next_button)
        time.sleep(2)  # Wait for the page to load
    except Exception as e:
        print(f"No more pages or unable to find the next button. Stopping. Error: {e}")
        break
    
    # Increment the page counter
    current_page += 1

# Close the WebDriver
driver.quit()

# Create a DataFrame from the list of data
if data:
    print(f"Data collected: {data}")  # Debugging statement
    df = pd.DataFrame(data[1:], columns=data[0])  # Use the first row as header
    df = df.dropna(how='all')  # Drop rows where all elements are NaN
else:
    df = pd.DataFrame()

# Display the DataFrame
print(df)

The result is always: No more pages or unable to find the next button. Stopping. Error: Message:

Data collected: [[‘Stazione’, ‘Indirizzo’, ‘Comune’, ‘Provincia’, ‘Classificazione’], [‘Abano Terme’, ‘Via Della Stazione, 10’, ‘Abano Terme’…. (page 1 of the table)