I have some code I open up a web page and then I try to set focus on a test box so a user can start typing, however when the page opens and the cursor is flashing on the correct text box if the user starts typing the URL bar is where the typing will occur how do I get it to show in the text box?
# import the required libraries
import undetected_chromedriver as uc
# define Chrome options
options = uc.ChromeOptions()
# set headless to False to run in non-headless mode
options.headless = False
# set up uc.Chrome(use_subprocess=True, options=options)
from arcgis.gis import GIS
from arcgis.geometry import Point, Polyline, Polygon
import datetime
import os
from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
import time
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
#driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()))
driver = uc.Chrome(use_subprocess=True, options=options)
driver.get("https://library.usask.ca/#gsc.tab=0")
q_field = driver.find_element("id", "primoQueryTemp")
q_field.send_keys("_")
q_field.click()
time.sleep(566)
what I tried is in there
find_element
send_keys
and even
.click()
but it is still defaulting to URL box
3
Actions.moveToElement() technically provides just hover effect though it works for all the input fields.
sendKeys("") may not work with all the types of inputs
In some cases, using JavaScript (element.focus()) along with Selenium is the most reliable way to focus on an element even during testing.
# Import the required libraries
import undetected_chromedriver as uc
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from webdriver_manager.chrome import ChromeDriverManager
import time
# Define Chrome options
options = uc.ChromeOptions()
# Set headless to False to run in non-headless mode
options.headless = False
# Chrome driver
driver = uc.Chrome(use_subprocess=True, options=options)
# Go to the desired URL
driver.get("https://library.usask.ca/#gsc.tab=0")
# Wait until the input field is visible
wait = WebDriverWait(driver, 10)
q_field = wait.until(EC.presence_of_element_located((By.ID, "primoQueryTemp")))
# Use JavaScript to focus the element
driver.execute_script("arguments[0].focus();", q_field)
# Initiate typing into the field
q_field.send_keys("_")
# Click the field to trigger any other events
q_field.click()
# Keep the browser open for observation
time.sleep(566)
# Close the driver
driver.quit()
2
I’m not sure what’s causing your issue but I’m assuming it’s some of the chrome options and/or the undetected_chromedriver. It could also be that you have a mismatched driver. The code below works and will also handle downloading the current chromedriver for you.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
driver = webdriver.Chrome()
driver.get("https://library.usask.ca/#gsc.tab=0")
wait = WebDriverWait(driver, 10)
driver.find_element(By.ID, "primoQueryTemp").send_keys("elon musk")
driver.find_element(By.CSS_SELECTOR, "button[title='Search']").click()
links = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "h3 > a")))
for link in links:
print(link.text)
and it outputs
Elon Musk
Elon Musk : Tesla, SpaceX, and the quest for a fantastic future
TEDTalks, Elon Musk—The mind behind Tesla, SpaceX, SolarCity ...
Elon Musk and Tesla : an electrifying love affair
The last days of twitter? : Elon Musk's calamitous takeover
Elon Musk's Twitter Takeover
Is Elon Musk Killing Twitter?, A Debate
Elon Musk and Twitter : a saga of tweets
Elon Musk and Twitter : a hard pill to swallow
Once a physicist: Elon Musk
This is complete working solution to scrape search results from the University of Saskatchewan Library website. The script inputs a search query (“artificial intelligence”) into the search bar, submits the search, and extracts details (e.g., item number, media type, image URL, and title, etc.) from the first 10 results.
To minimize detection by the website, I added stealthy Chrome options, such as disabling AutomationControlled features, setting a custom user-agent, and using –disable-extensions, etc.
Feel free to try it out and let me know if that’s what you’re trying to implement.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
# Initialize an empty list to store scraped data
data = []
# Function to configure Chrome options for stealth scraping
def get_stealth_chrome_options():
options = Options()
# Set headless mode (optional, uncomment to avoid loading browser UI)
# options.add_argument("--headless=new")
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_argument("--disable-extensions")
options.add_argument("--disable-infobars")
options.add_argument("--disable-popup-blocking")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--remote-debugging-port=9222")
options.add_argument("--window-size=1920,1080")
options.add_argument(
"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36")
# Suppress logging to reduce unnecessary output
options.add_argument("--log-level=3")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
# Ensure better resource handling for long scripts
options.add_argument("--disable-gpu")
options.add_argument("--enable-logging")
return options
# Set up the WebDriver with configured options
service = Service(ChromeDriverManager().install())
options = get_stealth_chrome_options()
browser = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(browser, 10)
try:
# Navigate to the target website
browser.get("https://library.usask.ca/#gsc.tab=0")
print("[INFO] Successfully loaded the website.")
# Locate the search field and input query
q_field = browser.find_element(By.ID, "primoQueryTemp")
q_field.send_keys("artificial intelligence")
q_field.send_keys(Keys.ENTER)
print("[INFO] Search query submitted.")
# Wait for the search results container to be visible
results_container = wait.until(
EC.presence_of_element_located((By.ID, "searchResultsContainer"))
)
print("[INFO] Search results container loaded.")
# Scrape the first 10 search results
for i in range(1, 11):
try:
# Locate each search result container by its XPath
container = results_container.find_element(By.XPATH, f"//*[@id='searchResultsContainer']/div[{i}]")
# Extract relevant information for each result
item_data = {
"item_number": container.find_element(By.CLASS_NAME, "list-item-count").text,
"media_type": container.find_element(By.CSS_SELECTOR, "div.media-content-type.align-self-start").text,
"image": container.find_element(By.CLASS_NAME, "media-thumbnail")
.find_element(By.CSS_SELECTOR, "div:nth-child(1) > img")
.get_attribute("src"),
"item_title": container.find_element(By.CLASS_NAME, "item-title").text,
}
data.append(item_data)
# print(f"[INFO] Scraped item {i}: {item_data}")
except Exception as e:
print(f"[WARNING] Error scraping item {i}: {e}")
# Print the collected data
print("[INFO] Scraping completed successfully.")
print(data)
except Exception as e:
print(f"[ERROR] An error occurred: {e}")
finally:
# Ensure the browser is properly closed
browser.quit()
print("[INFO] Browser closed.")
output:
[INFO] Successfully loaded the website.
[INFO] Search query submitted.
[INFO] Search results container loaded.
[INFO] Scraping completed successfully.
[{'item_number': '1', 'media_type': 'PRINT', 'image': 'https://proxy-ca.hosted.exlibrisgroup.com/exl_rewrite/syndetics.com/index.php?client=primo&isbn=9781633697898/lc.jpg', 'item_
title': 'Artificial intelligence'}, {'item_number': '2', 'media_type': 'PRINT', 'image': 'https://proxy-ca.hosted.exlibrisgroup.com/exl_rewrite/syndetics.com/index.php?client=primo
&isbn=9781682178676/lc.jpg', 'item_title': 'Artificial intelligence'}, {'item_number': '3', 'media_type': 'JOURNAL', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/i
con_journal.png', 'item_title': 'Artificial intelligence.'}, {'item_number': '4', 'media_type': 'MULTIPLE VERSIONS', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/i
con_versions.png', 'item_title': 'Artificial intelligence'}, {'item_number': '5', 'media_type': 'PRINT', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/icon_other.pn
g', 'item_title': 'Artificial intelligence'}, {'item_number': '6', 'media_type': 'PRINT', 'image': 'https://proxy-ca.hosted.exlibrisgroup.com/exl_rewrite/books.google.com/books/con
tent?id=K4lQAAAAMAAJ&printsec=frontcover&img=1&zoom=5', 'item_title': 'Artificial intelligence'}, {'item_number': '7', 'media_type': 'PRINT', 'image': 'https://proxy-ca.hosted.exli
brisgroup.com/exl_rewrite/books.google.com/books/content?id=xBlwX6TrBQoC&printsec=frontcover&img=1&zoom=5', 'item_title': 'Artificial intelligence'}, {'item_number': '8', 'media_ty
pe': 'MULTIPLE VERSIONS', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/icon_versions.png', 'item_title': 'Artificial intelligence'}, {'item_number': '9', 'media_ty
pe': 'MULTIPLE VERSIONS', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/icon_versions.png', 'item_title': 'Artificial intelligence.'}, {'item_number': '10', 'media_type': 'PROJECTED MEDIUM', 'image': 'https://usask.primo.exlibrisgroup.com/discovery/img/icon_other.png', 'item_title': 'Artificial Intelligence'}]
[INFO] Browser closed.