My result is empty not sure what went wrong with my code:
Here my code:
#trying to grab data from this site https://www.watsons.com.sg/health/c/2100000?currentPage=1
<code>import requests
from bs4 import BeautifulSoup
import pandas as pd
# Setting display options for pandas
pd.options.display.width = 1000
pd.options.display.max_rows = 1000
</code>
<code>import requests
from bs4 import BeautifulSoup
import pandas as pd
# Setting display options for pandas
pd.options.display.width = 1000
pd.options.display.max_rows = 1000
</code>
import requests
from bs4 import BeautifulSoup
import pandas as pd
# Setting display options for pandas
pd.options.display.width = 1000
pd.options.display.max_rows = 1000
Creating a User-Agent
<code>HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}
# Create lists
items = []
prices = []
# Function to scrape a page
def scrape_page(page):
try:
url = f'https://www.watsons.com.sg/health/c/2100000?currentPage={page}'
response = requests.get(url, headers=HEADERS)
response.raise_for_status() # Check for request errors
soup = BeautifulSoup(response.text, 'html.parser')
# Find product items
product_items = soup.find_all('e2-product-tile', class_='ng-star-inserted hasPromotion-2')
print(soup)
for item in product_items:
# Extract product name
name = item.find('h2', class_='productName').get_text(strip=True)
items.append(name)
# Extract product price
price = item.find('div', class_='formatted-value ng-star-inserted').get_text(strip=True)
prices.append(price)
except requests.RequestException as e:
print(f"An error occurred: {e}")
</code>
<code>HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}
# Create lists
items = []
prices = []
# Function to scrape a page
def scrape_page(page):
try:
url = f'https://www.watsons.com.sg/health/c/2100000?currentPage={page}'
response = requests.get(url, headers=HEADERS)
response.raise_for_status() # Check for request errors
soup = BeautifulSoup(response.text, 'html.parser')
# Find product items
product_items = soup.find_all('e2-product-tile', class_='ng-star-inserted hasPromotion-2')
print(soup)
for item in product_items:
# Extract product name
name = item.find('h2', class_='productName').get_text(strip=True)
items.append(name)
# Extract product price
price = item.find('div', class_='formatted-value ng-star-inserted').get_text(strip=True)
prices.append(price)
except requests.RequestException as e:
print(f"An error occurred: {e}")
</code>
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}
# Create lists
items = []
prices = []
# Function to scrape a page
def scrape_page(page):
try:
url = f'https://www.watsons.com.sg/health/c/2100000?currentPage={page}'
response = requests.get(url, headers=HEADERS)
response.raise_for_status() # Check for request errors
soup = BeautifulSoup(response.text, 'html.parser')
# Find product items
product_items = soup.find_all('e2-product-tile', class_='ng-star-inserted hasPromotion-2')
print(soup)
for item in product_items:
# Extract product name
name = item.find('h2', class_='productName').get_text(strip=True)
items.append(name)
# Extract product price
price = item.find('div', class_='formatted-value ng-star-inserted').get_text(strip=True)
prices.append(price)
except requests.RequestException as e:
print(f"An error occurred: {e}")
Scrape the first 5 pages as an example
<code>for page in range(1, 2):
scrape_page(page)
</code>
<code>for page in range(1, 2):
scrape_page(page)
</code>
for page in range(1, 2):
scrape_page(page)
Create a DataFrame
<code>df = pd.DataFrame({'Item': items, 'Price': prices})
</code>
<code>df = pd.DataFrame({'Item': items, 'Price': prices})
</code>
df = pd.DataFrame({'Item': items, 'Price': prices})
Display the DataFrame
<code>print(df)
</code>
<code>print(df)
</code>
print(df)
#The above is my code