The problem is that this website data is not showing up: https://www.medicalcouncil.ie/public-information/check-the-register/search-results/?regno=411252

It is currently just showing up if I take a screenshot or convert it to pdf but I’m struggling to actually access the html.

enter image description here

import asyncio
import pandas as pd
from playwright.async_api import async_playwright

async def main():
    async with async_playwright() as playwright:
        await run(playwright)

async def run(playwright):
    df = pd.DataFrame(columns=['salutation','sex','firstname','surname','speciality','registration_number','qualifications','date_of_qualification','education'])
    chromium = await playwright.chromium.launch(headless=False)
    page = await chromium.new_page()

    await page.goto("https://www.medicalcouncil.ie/public-information/check-the-register/search-results/?regno=411252")

    # Close the cookie consent button if it exists
    try:
        await page.click("#closeCookieButton")
    except Exception as e:
        print(f"Error clicking cookie button: {e}")

    # Wait for some time to ensure all content is loaded (replace with appropriate wait if needed)
    await page.wait_for_timeout(2000)  # Wait for 2 seconds

    await page.wait_for_load_state('networkidle')

    await page.wait_for_timeout(2000)  # Wait for 2 seconds

    await page.wait_for_load_state('networkidle')

    await page.set_viewport_size({"width": 1200, "height": 800})

    # Scroll to the bottom of the page to load more content if necessary
    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')

    # Wait for specific content to load, if needed (replace with appropriate wait logic)
    await page.wait_for_selector('#checkdetailsWrapper > div.ResultsTable.checkdetails')

    # Get the full content asynchronously
    content = await page.content()
    print(content)

    # Perform other actions if needed, such as taking screenshots or extracting data

    await chromium.close()

if __name__ == '__main__':
    asyncio.run(main())

I have tried to load the page properly by waiting for the following: long timeout, load state, changing viewport_size,headless false and true, scrolling. These all ran but when I wait for a specific selector such as #checkdetailsWrapper > div.ResultsTable.checkdetails it doesn’t load.