I’m trying to create a mini project where I need to use web crawler tools to gather up until 10,000 images. By far the script is fine until it keep giving the same results: ‘Found: 0 image links, looking for more …’

Is there any mistakes in my script that I overlooked it or did i miss anything? need urgent help as it’s due by next week.

Here is the script:

import os
import time
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

def fetch_image_urls(query, max_links_to_fetch, wd, sleep_between_interactions=1):
    def scroll_to_end(wd):
        wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(sleep_between_interactions)

    search_url = f"https://www.google.com/search?q={query}&tbm=isch"
    wd.get(search_url)

    image_urls = set()
    image_count = 0
    results_start = 0

    while image_count < max_links_to_fetch:
        scroll_to_end(wd)

        thumbnails = wd.find_elements(By.CSS_SELECTOR, "div[jsname='TVq1Bb'] img.rg_i")
        
        for img in thumbnails[results_start:]:
            try:
                img.click()
                time.sleep(sleep_between_interactions)
            except Exception as e:
                print(f"Error clicking image: {e}")
                continue

            large_image = wd.find_element(By.CSS_SELECTOR, "div[jsname='isl9D'] img.n3VNCb")
            image_url = large_image.get_attribute("src")

            if image_url and "http" in image_url:
                image_urls.add(image_url)

            image_count = len(image_urls)

            if len(image_urls) >= max_links_to_fetch:
                break
        else:
            print("Found:", len(image_urls), "image links, looking for more ...")
            time.sleep(30)
            continue
        
        results_start = len(thumbnails)

    return image_urls

def download_images(query, num_images, output_path):
    if not os.path.exists(output_path):
        os.makedirs(output_path)

    wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
    res = fetch_image_urls(query, num_images, wd=wd, sleep_between_interactions=1.5)

    for i, url in enumerate(res):
        try:
            image_content = requests.get(url).content
            with open(os.path.join(output_path, f'{query}_{i+1}.jpg'), 'wb') as f:
                f.write(image_content)
            print(f"Successfully downloaded {i+1}/{num_images} images")
        except Exception as e:
            print(f"Could not download {url} - {e}")

    wd.quit()

queries = ["medical imaging", "MRI scans", "CT scans", "X-ray images"]
for query in queries:
    download_images(query, num_images=500, output_path='medical_images')

Here the result:

Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …
Found: 0 image links, looking for more …