I am trying to scrape ebay sold listings on products that I have pulled from Lego to then work out average sold price of those listings.
The problem I am having is that it is including products that fall under “Results Matching fewer words” and those are not relevant to what I want to see.
I have tried to stop processing when it hits that selector but it then pulls through #N/As.
Can you help edit this for use in python?
<code>def fetch_ebay_average_price(ebay_search_url):
""" Fetch the average sold price and count of sold listings from eBay sold listings page. """
try:
service = Service(CHROME_DRIVER_PATH)
driver = webdriver.Chrome(service=service, options=chrome_options)
driver.get(ebay_search_url)
time.sleep(3)
soup = BeautifulSoup(driver.page_source, 'html.parser')
price_elements = soup.select('span.s-item__price')
prices = []
for price_element in price_elements:
try:
price_text = price_element.text.strip()
price_value = float(re.sub(r'[£,$,€]', '', price_text.split()[0].replace(',', '')))
prices.append(price_value)
except Exception as e:
print(f"Error processing price: {e}")
continue
driver.quit()
if prices:
average_price = round(statistics.mean(prices), 2)
sold_count = len(prices)
return f"£{average_price}", sold_count
else:
return "N/A", 0
except Exception as e:
print(f"Error fetching eBay average price: {e}")
return "N/A", 0
</code>
<code>def fetch_ebay_average_price(ebay_search_url):
""" Fetch the average sold price and count of sold listings from eBay sold listings page. """
try:
service = Service(CHROME_DRIVER_PATH)
driver = webdriver.Chrome(service=service, options=chrome_options)
driver.get(ebay_search_url)
time.sleep(3)
soup = BeautifulSoup(driver.page_source, 'html.parser')
price_elements = soup.select('span.s-item__price')
prices = []
for price_element in price_elements:
try:
price_text = price_element.text.strip()
price_value = float(re.sub(r'[£,$,€]', '', price_text.split()[0].replace(',', '')))
prices.append(price_value)
except Exception as e:
print(f"Error processing price: {e}")
continue
driver.quit()
if prices:
average_price = round(statistics.mean(prices), 2)
sold_count = len(prices)
return f"£{average_price}", sold_count
else:
return "N/A", 0
except Exception as e:
print(f"Error fetching eBay average price: {e}")
return "N/A", 0
</code>
def fetch_ebay_average_price(ebay_search_url):
""" Fetch the average sold price and count of sold listings from eBay sold listings page. """
try:
service = Service(CHROME_DRIVER_PATH)
driver = webdriver.Chrome(service=service, options=chrome_options)
driver.get(ebay_search_url)
time.sleep(3)
soup = BeautifulSoup(driver.page_source, 'html.parser')
price_elements = soup.select('span.s-item__price')
prices = []
for price_element in price_elements:
try:
price_text = price_element.text.strip()
price_value = float(re.sub(r'[£,$,€]', '', price_text.split()[0].replace(',', '')))
prices.append(price_value)
except Exception as e:
print(f"Error processing price: {e}")
continue
driver.quit()
if prices:
average_price = round(statistics.mean(prices), 2)
sold_count = len(prices)
return f"£{average_price}", sold_count
else:
return "N/A", 0
except Exception as e:
print(f"Error fetching eBay average price: {e}")
return "N/A", 0
Here is an example product from lego and it’s sold listings on ebay:
https://www.lego.com/en-gb/product/stable-of-dream-creatures-71459
https://www.ebay.co.uk/sch/i.html?_nkw=Lego%20Stable%20of%C2%A0Dream%C2%A0Creatures%2071459&rt=nc&LH_Sold=1&LH_Complete=1
When i inspect the ebay page, I focused on the following element:
srp-river-answer–REWRITE_START” element when inspecting the ebay page
New contributor
click90 is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.
2
It’s probably best to select all items displayed in the search results list first. Then you can loop over the items, grab their prices, and stop when you encounter the separator “Results matching fewer words”.
The following worked for me:
<code>def fetch_ebay_average_price(url):
options = webdriver.ChromeOptions()
options.add_argument('--headless=new')
try:
driver = webdriver.Chrome(options=options)
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'html.parser')
items = soup.select('.srp-list li')
for item in items:
if 'srp-river-answer--REWRITE_START' in item.get('class', ''):
notice = item.select_one('.section-notice__main')
print(f'Found separator "{notice.text}". Stopping.')
break
name = item.select_one('.s-item__title')
price = item.select_one('.s-item__price')
if name and price:
print(f'Found item "{name.text}" with price {price.text}')
finally:
driver.close()
</code>
<code>def fetch_ebay_average_price(url):
options = webdriver.ChromeOptions()
options.add_argument('--headless=new')
try:
driver = webdriver.Chrome(options=options)
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'html.parser')
items = soup.select('.srp-list li')
for item in items:
if 'srp-river-answer--REWRITE_START' in item.get('class', ''):
notice = item.select_one('.section-notice__main')
print(f'Found separator "{notice.text}". Stopping.')
break
name = item.select_one('.s-item__title')
price = item.select_one('.s-item__price')
if name and price:
print(f'Found item "{name.text}" with price {price.text}')
finally:
driver.close()
</code>
def fetch_ebay_average_price(url):
options = webdriver.ChromeOptions()
options.add_argument('--headless=new')
try:
driver = webdriver.Chrome(options=options)
driver.get(url)
soup = BeautifulSoup(driver.page_source, 'html.parser')
items = soup.select('.srp-list li')
for item in items:
if 'srp-river-answer--REWRITE_START' in item.get('class', ''):
notice = item.select_one('.section-notice__main')
print(f'Found separator "{notice.text}". Stopping.')
break
name = item.select_one('.s-item__title')
price = item.select_one('.s-item__price')
if name and price:
print(f'Found item "{name.text}" with price {price.text}')
finally:
driver.close()
It prints a bunch of product names and prices, and then stops:
<code>...
Found item "LEGO Dreamzzz: Stable of Dream Creatures (71459) BRAND NEW FACTORY SEALED" with price £28.55
Found item "LEGO DREAMZZZ: Stable of Dream Creatures (71459) 681 Pcs. New In Box" with price £36.16
Found item "LEGO STICKER SHEET for 71459 Stable of Dream Creatures, NEW & Genuine!" with price £6.65
Found separator "Results matching fewer words". Stopping.
</code>
<code>...
Found item "LEGO Dreamzzz: Stable of Dream Creatures (71459) BRAND NEW FACTORY SEALED" with price £28.55
Found item "LEGO DREAMZZZ: Stable of Dream Creatures (71459) 681 Pcs. New In Box" with price £36.16
Found item "LEGO STICKER SHEET for 71459 Stable of Dream Creatures, NEW & Genuine!" with price £6.65
Found separator "Results matching fewer words". Stopping.
</code>
...
Found item "LEGO Dreamzzz: Stable of Dream Creatures (71459) BRAND NEW FACTORY SEALED" with price £28.55
Found item "LEGO DREAMZZZ: Stable of Dream Creatures (71459) 681 Pcs. New In Box" with price £36.16
Found item "LEGO STICKER SHEET for 71459 Stable of Dream Creatures, NEW & Genuine!" with price £6.65
Found separator "Results matching fewer words". Stopping.