I am currently developing a web application using Blazor.

And publishing Blazor through Synology’s Docker and also planning to use Selenium.

Due to certain constraints, I have to use the headless option with “options.AddArgument(“–headless”);”

While this works fine for most websites and I can retrieve the page source, for some specific websites, I only get “”.

This issue only occurs when using the headless option. Is there a solution to this problem?

(Using Version 127.0 of Selenium Chrome )

 private async Task<Dictionary<string, string>> UsingSelenium(string url)
        {
            var options = new ChromeOptions();
            options.AddArgument("--headless"); // headless mode
            options.AddArgument("--disable-gpu");
            options.AddArgument("--no-sandbox");
            options.AddArgument("--disable-dev-shm-usage");
            options.AddArgument("--disable-extensions");
            options.AddArgument("--start-maximized");
            options.AddArgument("User-Agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/118.0.2088.76");
            using var driver = new ChromeDriver(options);
            try
            {
                driver.Manage().Timeouts().PageLoad = _pageLoadTimeout;
                driver.Manage().Timeouts().AsynchronousJavaScript = _scriptTimeout;
                driver.Navigate().GoToUrl(url);

              
                await WaitForPageLoad(driver);

                
             var pageSource = (string)((IJavaScriptExecutor)driver).ExecuteScript("return document.documentElement.outerHTML");

                var doc = new HtmlDocument();
                doc.LoadHtml(pageSource);

            catch (Exception e)
            {
                Console.WriteLine($"Selenium error: {e.Message}");
            }
            finally
            {
                driver.Quit();
            }
            return null;
        }

        
        private async Task WaitForPageLoad(IWebDriver driver)
        {
            var wait = new WebDriverWait(driver, _pageLoadTimeout);

            // Wait for the initial page load
            wait.Until(d => ((IJavaScriptExecutor)d).ExecuteScript("return document.readyState").Equals("complete"));

            // Wait for any animations to complete (assumes animations are done using CSS transitions)
            wait.Until(d => (bool)((IJavaScriptExecutor)d).ExecuteScript(@"
            var animations = document.getAnimations();
            return animations.length === 0 || animations.every(a => a.playState === 'finished');
        "));

            // Give a short pause to ensure everything is settled
            await Task.Delay(1000);
        }

I tried using the Selenium headless=new option, but it still doesn’t work on certain websites