I am scraping data from the ‘futbin.com’ website. My code was working until recently, but after some changes were made to the site, it stopped working. Now, it only saves an empty array as JSON data, whereas I was previously able to successfully retrieve player data. What do I need to update in my code?
<code>import bs4
import cloudscraper as cloudscraper
import json
import unicodedata
datas = []
links = []
products = []
def normalize_char(char):
return unicodedata.normalize('NFKD', char).encode('ascii', 'ignore').decode('utf-8')
pageCount = 1
cardVerison = 'icons'
scraper = cloudscraper.create_scraper()
req = scraper.get('https://www.futbin.com/players?page=' + str(pageCount) + '&version=' + cardVerison + '&pos_type=all')
soup = bs4.BeautifulSoup(req.text, 'html.parser')
numbers = [1, 2]
for number in numbers:
players = soup.findAll('tr', 'player_tr_' + str(number))
for p in players:
tds = p.findAll('td')
name = tds[1].text.strip()
position = tds[3].text.strip()
overall = tds[2].text.strip()
pac = tds[9].text.strip()
sho = tds[10].text.strip()
pas = tds[11].text.strip()
dri = tds[12].text.strip()
defa = tds[13].text.strip()
phy = tds[14].text.strip()
image = p.findAll('img')
y = image[0]["data-original"]
if y and "players" in y:
p.imageURL = y
x = p.select('img[src^="https://cdn.futbin.com/content/fifa24/img/"]')
club_image = x[0]['src']
nationality_image = x[1]['src']
league_image = x[2]['src']
p.plName = ''.join((name)).strip()
clean_name = normalize_char(name)
p.plPosition = ''.join((position)).split("n")[0]
p.plOverall = ''.join((overall)).strip()
clean_overall = int(p.plOverall)
print('PAC: ', pac)
print('SHO: ', sho)
print('PAS: ', pas)
print('DRI: ', dri)
print('DEF: ', defa)
print('PHY: ', phy)
datas.append([p.imageURL, clean_name, p.plPosition, clean_overall])
products.append({
'player_name': clean_name,
'player_overall': clean_overall,
'player_position': p.plPosition,
'player_image': p.imageURL,
'player_club': club_image,
'player_nationality': nationality_image,
'player_league': league_image
})
print('worked!')
print(datas)
with open('demo' + ".json", "w", encoding='utf8') as outfile:
json.dump(products, outfile, ensure_ascii=False)
</code>
<code>import bs4
import cloudscraper as cloudscraper
import json
import unicodedata
datas = []
links = []
products = []
def normalize_char(char):
return unicodedata.normalize('NFKD', char).encode('ascii', 'ignore').decode('utf-8')
pageCount = 1
cardVerison = 'icons'
scraper = cloudscraper.create_scraper()
req = scraper.get('https://www.futbin.com/players?page=' + str(pageCount) + '&version=' + cardVerison + '&pos_type=all')
soup = bs4.BeautifulSoup(req.text, 'html.parser')
numbers = [1, 2]
for number in numbers:
players = soup.findAll('tr', 'player_tr_' + str(number))
for p in players:
tds = p.findAll('td')
name = tds[1].text.strip()
position = tds[3].text.strip()
overall = tds[2].text.strip()
pac = tds[9].text.strip()
sho = tds[10].text.strip()
pas = tds[11].text.strip()
dri = tds[12].text.strip()
defa = tds[13].text.strip()
phy = tds[14].text.strip()
image = p.findAll('img')
y = image[0]["data-original"]
if y and "players" in y:
p.imageURL = y
x = p.select('img[src^="https://cdn.futbin.com/content/fifa24/img/"]')
club_image = x[0]['src']
nationality_image = x[1]['src']
league_image = x[2]['src']
p.plName = ''.join((name)).strip()
clean_name = normalize_char(name)
p.plPosition = ''.join((position)).split("n")[0]
p.plOverall = ''.join((overall)).strip()
clean_overall = int(p.plOverall)
print('PAC: ', pac)
print('SHO: ', sho)
print('PAS: ', pas)
print('DRI: ', dri)
print('DEF: ', defa)
print('PHY: ', phy)
datas.append([p.imageURL, clean_name, p.plPosition, clean_overall])
products.append({
'player_name': clean_name,
'player_overall': clean_overall,
'player_position': p.plPosition,
'player_image': p.imageURL,
'player_club': club_image,
'player_nationality': nationality_image,
'player_league': league_image
})
print('worked!')
print(datas)
with open('demo' + ".json", "w", encoding='utf8') as outfile:
json.dump(products, outfile, ensure_ascii=False)
</code>
import bs4
import cloudscraper as cloudscraper
import json
import unicodedata
datas = []
links = []
products = []
def normalize_char(char):
return unicodedata.normalize('NFKD', char).encode('ascii', 'ignore').decode('utf-8')
pageCount = 1
cardVerison = 'icons'
scraper = cloudscraper.create_scraper()
req = scraper.get('https://www.futbin.com/players?page=' + str(pageCount) + '&version=' + cardVerison + '&pos_type=all')
soup = bs4.BeautifulSoup(req.text, 'html.parser')
numbers = [1, 2]
for number in numbers:
players = soup.findAll('tr', 'player_tr_' + str(number))
for p in players:
tds = p.findAll('td')
name = tds[1].text.strip()
position = tds[3].text.strip()
overall = tds[2].text.strip()
pac = tds[9].text.strip()
sho = tds[10].text.strip()
pas = tds[11].text.strip()
dri = tds[12].text.strip()
defa = tds[13].text.strip()
phy = tds[14].text.strip()
image = p.findAll('img')
y = image[0]["data-original"]
if y and "players" in y:
p.imageURL = y
x = p.select('img[src^="https://cdn.futbin.com/content/fifa24/img/"]')
club_image = x[0]['src']
nationality_image = x[1]['src']
league_image = x[2]['src']
p.plName = ''.join((name)).strip()
clean_name = normalize_char(name)
p.plPosition = ''.join((position)).split("n")[0]
p.plOverall = ''.join((overall)).strip()
clean_overall = int(p.plOverall)
print('PAC: ', pac)
print('SHO: ', sho)
print('PAS: ', pas)
print('DRI: ', dri)
print('DEF: ', defa)
print('PHY: ', phy)
datas.append([p.imageURL, clean_name, p.plPosition, clean_overall])
products.append({
'player_name': clean_name,
'player_overall': clean_overall,
'player_position': p.plPosition,
'player_image': p.imageURL,
'player_club': club_image,
'player_nationality': nationality_image,
'player_league': league_image
})
print('worked!')
print(datas)
with open('demo' + ".json", "w", encoding='utf8') as outfile:
json.dump(products, outfile, ensure_ascii=False)
1