I am trying to build a scrapper for the US Gucci website. I do manage to get the data for the products on the first page of the handbags category, however, after clicking on the load more button and scrolling to the bottom of the page I get the error: ‘NoneType’ object has no attribute ‘text’
Here is the code I have written:
try:
# Initializing men_product list
handbags_us_list = []
dtime = datetime.now().strftime('%Y-%m-%d')
brand_name = "Gucci"
country_name = "United States"
while True:
soup = BeautifulSoup(driver.page_source, 'html.parser')
handbags_wm = soup.find_all('div', {'class': 'product-tiles-grid-item-detail'})
for bags in handbags_wm:
#print(mens)
bag_name = bags.find('h2').text.strip()
bag_price = bags.find('span', {'class': 'sale'}).text.strip()
bag_id = bags.find('div', {'class': 'carousel product-tiles-grid-item-carousel'})['data-product-code']
print(bag_name)
print(bag_price)
print(bag_id)
# Try to click the "Load More" button
try:
#Load more products
load_all = safe_click_js(driver, "//a[@class='cta is-primary ajax-loader-link product-tiles-grid-load']")
time.sleep(10)
if load_all:
print("Load More Button clicked successfully!")
# For example, check if the URL changes or a specific element appears
time.sleep(5) # Wait for the action to complete
else:
print("Failed to click the button after multiple attempts.")
#Keep Scrolling until all the products are loaded
element = find_element_with_retry(driver, "label-for-mobile_store_locator_region")
driver.execute_script("arguments[0].scrollIntoView(true);", element)
time.sleep(40)
except (TimeoutException, NoSuchElementException):
print("No more 'Load More' button found or it is not clickable.")
break