We have created a script for automated site tests in selenium, the test detects whether the correct cookies are loaded when the consent is selected in the cookie bar cookiebot.

Example, the script in the cookie bar clicks on the consent for marketing and analytics cookies and then checks if the cookies allowed for this consent are loaded. We check cookies at: https://innogy.cz

The script uses headless mode and unfortunately we are not able to load the identical cookie list as a real user. We run the script in Azure DevOps in the docker.

Here is the script:

import os
import time
import pandas as pd
import smtplib
import requests
from io import StringIO
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from webdriver_manager.chrome import ChromeDriverManager
import re

# Function to download and retrieve cookies from Google Spreadsheet
def load_cookies_from_public_csv(url):
    sheet_id = url.split('/')[5]
    dwn_url = f'https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv'
    response = requests.get(dwn_url)
    if response.status_code == 200:
        csv_content = response.content.decode('utf-8')
        df = pd.read_csv(StringIO(csv_content))
        return df
    else:
        raise Exception(f "Could not read CSV file. HTTP Status code: {response.status_code}")

# Function to send a Gmail SMTP email with an attached image
def send_email(subject, body, to_emails, screenshot_path):
    from_email = os.getenv("EMAIL_ADDRESS")
    from_password = os.getenv("EMAIL_PASSWORD")

    msg = MIMEMultipart()
    msg['From'] = from_email
    msg['To'] = ", ".join(to_emails)
    msg['Subject'] = subject
    
    msg.attach(MIMEText(body, 'html'))  # Using HTML format

    # Attaching a screenshot
    with open(screenshot_path, 'rb') as f:
        img_data = f.read()
        image = MIMEImage(img_data, name=os.path.basename(screenshot_path))
        msg.attach(image)

    try:
        # Using port 465 for SSL/TLS connection to Gmail SMTP server
        server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
        server.login(from_email, from_password)
        text = msg.as_string()
        server.sendmail(from_email, to_emails, text)
        server.quit()
        print("Email sent successfully")
    except Exception as e:
        print(f "Error sending email: {e}")

# Helper function to create HTML list
def create_html_list(items):
    return "<ul>" + "".join(f"<li>{item}</li>" for item in items) + "</ul>"

# Cookie comparison function
def compare_cookies(loaded_cookies, allowed_cookies):
    allowed = set(allowed_cookies)
    disallowed_cookies = { cookie for cookie in loaded_cookies if cookie not in allowed and not re.match(r'^TSd+', cookie)}
    return disallowed_cookies

# Initializing the web browser
options = Options()
options.add_argument("--no-sandbox")
options.add_argument("--allow-persistent-cookies")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1920,1080")
options.add_argument("--disable-extensions")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--headless=new")
options.add_argument("--user-data-dir=/tmp/user_data")
options.add_argument("--disable-background-timer-throttling")
options.add_argument("--disable-renderer-backgrounding")
options.add_argument("--disable-backgrounding-occluded-windows")
options.add_argument("--disable-background-networking")
options.add_argument("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.57 Safari/537.36")

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

try:
    print("Clearing all cookies...")
    driver.delete_all_cookies()

    print("Opening a web page...")
    driver.get("https://innogy.cz")

    print("Refreshing the page...")
    driver.refresh()

    print("Waiting for page to load and cookie consent bar...")
    time.sleep(10)

    preference_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="c-content"]/div/div[2]/label/span')))
    print("Clicking the 'Preference' switch...")
    driver.execute_script("arguments[0].click();", preference_button)
    time.sleep(5)

    statistic_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="c-content"]/div/div[3]/label/span')))
    print("Clicking the 'Analytics' switch...")
    driver.execute_script("arguments[0].click();", statistic_button)
    time.sleep(5)

    marketing_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="c-content"]/div/div[4]/label/span')))
    print("Clicking the 'Marketing' switch...")
    driver.execute_script("arguments[0].click();", marketing_button)
    time.sleep(5)

    # Simulate mouse movement
    actions = ActionChains(driver)
    actions.move_by_offset(10, 10).perform()

    allow_selection_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll"]'))
    print("Clicking the 'Enable Selection' button...")
    driver.execute_script("arguments[0].click();", allow_selection_button)
    time.sleep(10)

    print("Getting the cookie list...")
    cookies = driver.get_cookies()
    cookies_names = [cookie['name'] for cookie in cookies]
    print("Cookies found:")
    for cookie in cookies_names:
        print(cookie)

    print("Adding a new cookie...")
    driver.add_cookie({'name': 'test_cookie', 'value': 'test_value'})

    print("Checking local storage...")
    local_storage_item = driver.execute_script("return localStorage.getItem('test_key');")
    print(local_storage_item)

    csv_url = "https://docs.google.com/spreadsheets/d/1x5Wkvpu4imykcz1KHogDUPHYAP87kMfF--j2gd2bDX4/export?format=csv"
    df = load_cookies_from_public_csv(csv_url)
    allowed_cookies = df[df['Category'].isin(['Preferences', 'Necessary'])]['Name'].tolist()

    print("Comparing cookie lists...")
    disallowed_cookies = compare_cookies(cookies_names, allowed_cookies)

    print("nResult of checking cookies:")
    if not disallowed_cookies:
        result = "The loaded cookies are consistent with the list of cookies that can be loaded.<br>Screenshot is taken after the cookies are loaded and declares what type of consent was selected."
    else:
        result = f "Warning, the following disallowed cookies have been loaded: {', '.join(disallowed_cookies)}"

    print(result)

    footer_link = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/footer/div[1]/div[2]/div/div[1]/div/ul/li[4]/a')))
    print("Clicking on the 'Cookies settings' link in the footer...")
    driver.execute_script("arguments[0].click();", footer_link)
    time.sleep(10)

    cookiebanner = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, '//*[@id="cookiebanner"]')))
    preference_button = driver.find_element(By.XPATH, '//*[@id="c-content"]/div/div[2]')

    screenshot_path = "screenshot_with_consent.png"
    driver.save_screenshot(screenshot_path)

    email_body = f"""
    <html>
    <body>
        <p>This is a notification of a cookie control test on the innogy.cz website.</p>
        <p>Consent type used: <strong>Preference</strong>.</p>
        <p><strong>The following cookies were loaded:</strong><br>{create_html_list(cookies_names)}</p>
        <p>{result}</p>
        <p>Attached screenshot of the page is attached.</p>
    </body>
    </html>
    """

    recipient_list=["[email protected]", "[email protected]"]
    send_email("Cookie Control Test Notification - Preference", email_body, recipient_list, screenshot_path)

Finally:
    driver.quit()

I need to know what method/code modification would achieve loading the correct list of cookies