I wrote a script to list the venues of a city from a website. However, I encountered two issues that I don’t know how to solve:

1. Although I wrote code to scroll, it only captures the first 15 items and doesn’t list the rest.

2. The script doesn’t click the “Next” button to go to the next page.

   Plain text

   Copy to clipboard

   Open code in new window

   EnlighterJS 3 Syntax Highlighter

   <code>place='Abu Dhabi'

   #----------------------------------------------- Cvent

   url="https://www-eur.cvent.com/venues/"

   driver = webdriver.Chrome()

   driver.get(url)

   driver.maximize_window()

   search_box = driver.find_element(By.CSS_SELECTOR, 'input#searchString')

   search_box.send_keys(place)

   search_box.send_keys(Keys.RETURN)

   time.sleep(5)

   visited_links = {}

   Titles = []

   pages=[]

   while True:

   driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

   cventlinks = []

   link_elements = driver.find_elements(By.CSS_SELECTOR,"li.w-full a")

   for link_element in link_elements:

   cventlinks.append(link_element.get_attribute("href"))

   Title_elements = driver.find_elements(By.CSS_SELECTOR,'li.w-full h3')

   for item in Title_elements:

   Titles.append(item.text)

   with open('CventLinks.csv', 'a', newline='') as csvfile:

   writer = csv.writer(csvfile)

   for title in Titles:

   if title not in visited_links:

   writer.writerow([cventlinks[Titles.index(title)]])

   visited_links[title] = 1

   next_button=driver.find_elements(By.CSS_SELECTOR,'a[aria-label*="Next page"]')

   ActionChains(driver).move_to_element(next_button[0]).click().perform()

   time.sleep(5)

   if driver.current_url not in pages:

   pages.append(driver.current_url)

   else:

   break

   </code>

   <code>place='Abu Dhabi' #----------------------------------------------- Cvent url="https://www-eur.cvent.com/venues/" driver = webdriver.Chrome() driver.get(url) driver.maximize_window() search_box = driver.find_element(By.CSS_SELECTOR, 'input#searchString') search_box.send_keys(place) search_box.send_keys(Keys.RETURN) time.sleep(5) visited_links = {} Titles = [] pages=[] while True: driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") cventlinks = [] link_elements = driver.find_elements(By.CSS_SELECTOR,"li.w-full a") for link_element in link_elements: cventlinks.append(link_element.get_attribute("href")) Title_elements = driver.find_elements(By.CSS_SELECTOR,'li.w-full h3') for item in Title_elements: Titles.append(item.text) with open('CventLinks.csv', 'a', newline='') as csvfile: writer = csv.writer(csvfile) for title in Titles: if title not in visited_links: writer.writerow([cventlinks[Titles.index(title)]]) visited_links[title] = 1 next_button=driver.find_elements(By.CSS_SELECTOR,'a[aria-label*="Next page"]') ActionChains(driver).move_to_element(next_button[0]).click().perform() time.sleep(5) if driver.current_url not in pages: pages.append(driver.current_url) else: break </code>

   place='Abu Dhabi'
   #----------------------------------------------- Cvent
   url="https://www-eur.cvent.com/venues/"
   driver = webdriver.Chrome()
   driver.get(url)
   driver.maximize_window()
   search_box = driver.find_element(By.CSS_SELECTOR, 'input#searchString')
   search_box.send_keys(place)
   search_box.send_keys(Keys.RETURN)
   time.sleep(5)
   visited_links = {}
   Titles = []
   pages=[]
   
   while True:
       driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
       cventlinks = []
       link_elements = driver.find_elements(By.CSS_SELECTOR,"li.w-full a")
       for link_element in link_elements:
           cventlinks.append(link_element.get_attribute("href"))
       Title_elements = driver.find_elements(By.CSS_SELECTOR,'li.w-full h3')
       for item in Title_elements:
           Titles.append(item.text)
       with open('CventLinks.csv', 'a', newline='') as csvfile:
           writer = csv.writer(csvfile)
           for title in Titles:
               if title not in visited_links:
                   writer.writerow([cventlinks[Titles.index(title)]])
                   visited_links[title] = 1
       next_button=driver.find_elements(By.CSS_SELECTOR,'a[aria-label*="Next page"]')
       ActionChains(driver).move_to_element(next_button[0]).click().perform()
       time.sleep(5)
       if driver.current_url not in pages:
           pages.append(driver.current_url)
       else:
           break
   

Could you please guide me?