from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import pandas as pd
import time
import re

def get_total_pages(driver):
    soup = BeautifulSoup(driver.page_source, "html.parser")
    total_pages_element = soup.find("b", class_="dxp-lead dxp-summary")
    if total_pages_element:
        text = total_pages_element.get_text(strip=True)
        match = re.search(r'Страница (d+) из (d+)', text)
        if match:
            total_pages = int(match.group(2))
            return total_pages
    return None


def is_big_table_navigation(soup):
    page_indicator = soup.find("b", class_="dxp-lead dxp-summary")
    if page_indicator:
        text = page_indicator.get_text(strip=True)
        match = re.search(r'Страница (d+) из (d+)', text)
        if match:
            return True
    return False

def navigate_to_starting_page(driver, start_page, total_pages):
    current_page = 1
    
    from_start = start_page - 1
    from_end = total_pages - start_page

    if from_start <= from_end:
        print(f"Navigating to page {start_page} from the start...")
        while current_page < start_page:
            try:
                next_button = WebDriverWait(driver, 20).until(
                    EC.element_to_be_clickable((By.XPATH, "//a[contains(@onclick,'PBN')]"))
                )
                next_button.click()
                current_page += 1
                print(f"Moved to page {current_page}")
                WebDriverWait(driver, 20).until(
                    EC.text_to_be_present_in_element((By.XPATH, "//b[@class='dxp-lead dxp-summary']"), f"Страница {current_page}")
                )
                time.sleep(2)
            except Exception as e:
                print(f"Error navigating to page {current_page}: {e}")
                driver.save_screenshot(f"error_page_{current_page}.png")
                time.sleep(5)
                continue
    else:
        print(f"Navigating to page {start_page} from the end...")
        try:
            last_page_button = WebDriverWait(driver, 20).until(
                EC.element_to_be_clickable((By.XPATH, f"//a[@onclick and text()='{total_pages}']"))
            )
            last_page_button.click()
            current_page = total_pages
            print(f"Moved to page {current_page}")
            WebDriverWait(driver, 20).until(
                EC.text_to_be_present_in_element((By.XPATH, "//b[@class='dxp-lead dxp-summary']"), f"Страница {current_page}")
            )
            time.sleep(2)
        except Exception as e:
            print(f"Error navigating to last page: {e}")
            driver.save_screenshot("error_last_page.png")
            return
        
        while current_page > start_page:
            try:
                prev_button = WebDriverWait(driver, 20).until(
                    EC.element_to_be_clickable((By.XPATH, "//a[contains(@onclick,'PBP')]"))
                )
                prev_button.click()
                current_page -= 1
                print(f"Moved to page {current_page}")
                WebDriverWait(driver, 20).until(
                    EC.text_to_be_present_in_element((By.XPATH, "//b[@class='dxp-lead dxp-summary']"), f"Страница {current_page}")
                )
                time.sleep(2)
            except Exception as e:
                print(f"Error navigating to page {current_page}: {e}")
                driver.save_screenshot(f"error_page_{current_page}.png")
                time.sleep(5)
                continue

    print(f"Arrived at the starting page {start_page}.")

def is_valid_row(row):
    first_field = row.find("td").get_text(strip=True)
    return bool(re.match(r"^DMd+$", first_field))

def is_small_table_with_buttons(table):
    rows = table.find_all("tr")
    if len(rows) < 5:
        if table.find("div", class_="dxpLite_SoftOrange") or table.find("div", class_="dxgvPagerBottomPanel_SoftOrange"):
            return True
    return False

def scrape_table_data(start_page, end_page):
    driver = None
    all_data = []
    max_columns = 0

    try:
        driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
        driver.maximize_window()
        driver.get("https://registru.dispozitive.amdm.gov.md/")

        total_pages = get_total_pages(driver)
        if total_pages is None:
            print("Unable to determine total pages. Exiting.")
            return

        navigate_to_starting_page(driver, start_page, total_pages)

        current_page = start_page

        while current_page <= end_page:
            print(f"Scraping page {current_page}...")
            time.sleep(1)

            soup = BeautifulSoup(driver.page_source, "html.parser")
            
            if not is_big_table_navigation(soup):
                print("Ignoring small table navigation.")
                retry_count = 0
                max_retries = 5
                page_changed = False

                while retry_count < max_retries and not page_changed:
                    try:
                        next_button = WebDriverWait(driver, 20).until(
                            EC.element_to_be_clickable((By.XPATH, "//a[contains(@onclick,'PBN')]"))
                        )
                        next_button.click()
                        current_page += 1
                        print(f"Moved to page {current_page}")
                        WebDriverWait(driver, 20).until(
                            EC.text_to_be_present_in_element((By.XPATH, "//b[@class='dxp-lead dxp-summary']"), f"Страница {current_page}")
                        )
                        page_changed = True
                    except Exception as e:
                        retry_count += 1
                        print(f"Error moving to next page (attempt {retry_count}/{max_retries}): {e}")
                        driver.save_screenshot(f"error_next_page_{current_page}.png")
                        time.sleep(5)
                        continue

                if not page_changed:
                    print(f"Failed to move to next page after {max_retries} attempts. Exiting loop.")
                    break
                
                continue

            tables = soup.find_all("table", class_="dxgvTable_SoftOrange")

            for table in tables:
                rows = table.find_all("tr", id=lambda x: x and x.startswith("ctl00_ctl00_ASPxSplitter1_Content_ContentSplitter_MainContent_ASPxGridView4_DXDataRow"))
                if len(rows) < 5 or not any(is_valid_row(row) for row in rows):
                    print("Skipping a small table.")
                    continue

                for row in rows:
                    if is_valid_row(row):
                        columns = row.find_all("td")
                        data = [column.get_text(strip=True) for column in columns]
                        all_data.append(data)
                        if len(columns) > max_columns:
                            max_columns = len(columns)

            if current_page == end_page:
                print("Reached the end page.")
                break

            retry_count = 0
            max_retries = 5
            page_changed = False

            while retry_count < max_retries and not page_changed:
                try:
                    next_button = WebDriverWait(driver, 20).until(
                        EC.element_to_be_clickable((By.XPATH, "//a[contains(@onclick,'PBN')]"))
                    )
                    next_button.click()
                    current_page += 1
                    print(f"Moved to page {current_page}")
                    WebDriverWait(driver, 20).until(
                        EC.text_to_be_present_in_element((By.XPATH, "//b[@class='dxp-lead dxp-summary']"), f"Страница {current_page}")
                    )
                    page_changed = True
                except Exception as e:
                    retry_count += 1
                    print(f"Error moving to next page (attempt {retry_count}/{max_retries}): {e}")
                    driver.save_screenshot(f"error_next_page_{current_page}.png")
                    time.sleep(5)
                    continue

            if not page_changed:
                print(f"Failed to move to next page after {max_retries} attempts. Exiting loop.")
                break

    except Exception as e:
        print(f"An error occurred: {e}")
    finally:
        if driver:
            driver.quit()

        if all_data:
            for i in range(len(all_data)):
                if len(all_data[i]) < max_columns:
                    all_data[i].extend([None] * (max_columns - len(all_data[i])))

            columns = [f"Column{i+1}" for i in range(max_columns)]

            df = pd.DataFrame(all_data, columns=columns)

            file_name = f"registru.dispozitive.page{start_page}-{end_page}.xlsx"
            df.to_excel(file_name, index=False)
            print(f"Data saved to {file_name}")
        else:
            print("No data scraped to save.")

def menu():
    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
    driver.maximize_window()
    driver.get("https://registru.dispozitive.amdm.gov.md/")
    total_pages = get_total_pages(driver)
    driver.quit()
    
    if total_pages is None:
        print("Unable to determine the total number of pages. Exiting.")
        return

    print(f"Total pages: {total_pages}")

    start_page = 1
    end_page = total_pages

    while True:
        print("n1. Scrap all pages")
        print("2. Set starting page for Scrap")
        print("3. Set ending page for Scrap")
        print("4. Start Scraping")
        print("5. Quit")
        choice = input("Enter your choice (1-5): ")

        if choice == "1":
            start_page = 1
            end_page = total_pages
            print(f"Set to scrape all pages from {start_page} to {end_page}.")
        elif choice == "2":
            start_page = int(input(f"Enter the starting page (1-{total_pages}): "))
            if start_page < 1 or start_page > total_pages:
                print(f"Invalid starting page. Please enter a number between 1 and {total_pages}.")
            else:
                print(f"Starting page set to {start_page}.")
        elif choice == "3":
            end_page = int(input(f"Enter the ending page ({start_page}-{total_pages}): "))
            if end_page < start_page or end_page > total_pages:
                print(f"Invalid ending page. Please enter a number between {start_page} and {total_pages}.")
            else:
                print(f"Ending page set to {end_page}.")
        elif choice == "4":
            print(f"Scraping from page {start_page} to {end_page}.")
            scrape_table_data(start_page, end_page)
        elif choice == "5":
            print("Quitting the program.")
            break
        else:
            print("Invalid choice. Please try again.")

menu()


if __name__ == "__main__":
    menu()