I am trying to scrape the sku and description on this site: https://www.dewalt.com/products/power-tools/
but, it wont scrape the desired elements despite the code being able to run. Does anyone know why? Am I grabbing the wrong elements?
<code>import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
from selenium.common.exceptions import NoSuchElementException
options = Options()
driver = uc.Chrome()
website = 'https://www.dewalt.com/products/power-tools/drills'
driver.get(website)
WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#product-list')))
prod_num = []
prod_desc = []
container = driver.find_element(By.CSS_SELECTOR, '#product-list')
for _ in range(4):
driver.execute_script("window.scrollBy(0, 2000);")
time.sleep(2)
skus = container.find_elements(By.CLASS_NAME, 'coh-inline-element product-sku')
descriptions = container.find_elements(By.CLASS_NAME, 'coh-link subtitle card-link product-title')
for sku in skus:
prod_num.append(sku.text)
for desc in descriptions:
prod_desc.append(desc.text)
driver.quit()
print(len(prod_num))
print(len(prod_desc))
# Create a DataFrame from the scraped data
df = pd.DataFrame({'code': prod_num, 'description': prod_desc})
# Save the DataFrame to a CSV file
df.to_csv('dewtest1.csv', index=False)
print(df)
</code>
<code>import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
from selenium.common.exceptions import NoSuchElementException
options = Options()
driver = uc.Chrome()
website = 'https://www.dewalt.com/products/power-tools/drills'
driver.get(website)
WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#product-list')))
prod_num = []
prod_desc = []
container = driver.find_element(By.CSS_SELECTOR, '#product-list')
for _ in range(4):
driver.execute_script("window.scrollBy(0, 2000);")
time.sleep(2)
skus = container.find_elements(By.CLASS_NAME, 'coh-inline-element product-sku')
descriptions = container.find_elements(By.CLASS_NAME, 'coh-link subtitle card-link product-title')
for sku in skus:
prod_num.append(sku.text)
for desc in descriptions:
prod_desc.append(desc.text)
driver.quit()
print(len(prod_num))
print(len(prod_desc))
# Create a DataFrame from the scraped data
df = pd.DataFrame({'code': prod_num, 'description': prod_desc})
# Save the DataFrame to a CSV file
df.to_csv('dewtest1.csv', index=False)
print(df)
</code>
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
from selenium.common.exceptions import NoSuchElementException
options = Options()
driver = uc.Chrome()
website = 'https://www.dewalt.com/products/power-tools/drills'
driver.get(website)
WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#product-list')))
prod_num = []
prod_desc = []
container = driver.find_element(By.CSS_SELECTOR, '#product-list')
for _ in range(4):
driver.execute_script("window.scrollBy(0, 2000);")
time.sleep(2)
skus = container.find_elements(By.CLASS_NAME, 'coh-inline-element product-sku')
descriptions = container.find_elements(By.CLASS_NAME, 'coh-link subtitle card-link product-title')
for sku in skus:
prod_num.append(sku.text)
for desc in descriptions:
prod_desc.append(desc.text)
driver.quit()
print(len(prod_num))
print(len(prod_desc))
# Create a DataFrame from the scraped data
df = pd.DataFrame({'code': prod_num, 'description': prod_desc})
# Save the DataFrame to a CSV file
df.to_csv('dewtest1.csv', index=False)
print(df)
New contributor
Ryan Houghton is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.