I am trying to scrape a website for a mini project and the data that I needed are hidden under the #Shadow-root tag of the HTML. I tried accessing it with selenium with the code below:
<code>def expand_shadow_element(element):
shadow_root = driver.execute_script('return arguments[0].shadowRoot', element)
return shadow_root
url = "https://new.abb.com/products/SK615502-D"
#Initializing the webdriver
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(executable_path="/Users/ritchevy/Desktop/scraping-glassdoor/chromedriver", options=options)
timeout = 10
wait = WebDriverWait(driver, timeout)
driver.set_window_size(1120, 1000)
driver.get(url)
root1 = driver.find_element(By.CSS_SELECTOR,"pis-products-details-attribute-groups")
shadow_root1 = expand_shadow_element(root1)
shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
</code>
<code>def expand_shadow_element(element):
shadow_root = driver.execute_script('return arguments[0].shadowRoot', element)
return shadow_root
url = "https://new.abb.com/products/SK615502-D"
#Initializing the webdriver
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(executable_path="/Users/ritchevy/Desktop/scraping-glassdoor/chromedriver", options=options)
timeout = 10
wait = WebDriverWait(driver, timeout)
driver.set_window_size(1120, 1000)
driver.get(url)
root1 = driver.find_element(By.CSS_SELECTOR,"pis-products-details-attribute-groups")
shadow_root1 = expand_shadow_element(root1)
shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
</code>
def expand_shadow_element(element):
shadow_root = driver.execute_script('return arguments[0].shadowRoot', element)
return shadow_root
url = "https://new.abb.com/products/SK615502-D"
#Initializing the webdriver
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(executable_path="/Users/ritchevy/Desktop/scraping-glassdoor/chromedriver", options=options)
timeout = 10
wait = WebDriverWait(driver, timeout)
driver.set_window_size(1120, 1000)
driver.get(url)
root1 = driver.find_element(By.CSS_SELECTOR,"pis-products-details-attribute-groups")
shadow_root1 = expand_shadow_element(root1)
shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
Upon execution its giving me this error
<code>---> 35 shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
36
AttributeError: 'dict' object has no attribute 'find_element'
</code>
<code>---> 35 shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
36
AttributeError: 'dict' object has no attribute 'find_element'
</code>
---> 35 shadow_container_root = shadow_root1.find_element(By.CSS_SELECTOR,"div")
36
AttributeError: 'dict' object has no attribute 'find_element'
Any idea how to resolve this?