I am using a data generator to fetch data for a specified time frame. The code eventually crashes with a segmentation fault. I’ve used faulthandler to get more information, but I’m still unsure what’s causing the error. I am really at my wit’s end here.

Environment:

Python version: 3.12.3 (tested with 3.11.8 aswell), conda env
Operating System: Ubuntu 22.04

Code

from datetime import datetime, timezone
import os
from dotenv import load_dotenv
import httpx
from httpx import Timeout, HTTPStatusError, RequestError
import time


import faulthandler

file = open("faulthandler.log", "w")
faulthandler.enable(file=file, all_threads=True)


# Load environment variables
load_dotenv()
api_key = os.getenv("POLYGON_API_KEY")


def convert_timestamp(ts):
    """Convert Unix timestamp in nanoseconds to a formatted datetime string."""
    return datetime.fromtimestamp(ts / 1e9, tz=timezone.utc).strftime(
        "%Y-%m-%d %H:%M:%S"
    )


def polygon_trades_streamer(
    ticker,
    timestamp_gte,
    timestamp_lte,
    limit=5000,
    sort="timestamp",
    order="asc",
    api_key="your_api_key_here",
    max_retries=5,  # Maximum number of retries
    initial_backoff=1,  # Initial backoff duration in seconds
):
    base_url = f"https://api.polygon.io/v3/trades/{ticker}"
    params = {
        "timestamp.gte": timestamp_gte,
        "timestamp.lte": timestamp_lte,
        "limit": limit,
        "sort": sort,
        "order": order,
        "apiKey": api_key,
    }

    # Setting a timeout for connect and read operations
    timeout = Timeout(10)
    client = httpx.Client(timeout=timeout)

    try:
        while True:
            retry_count = 0
            while retry_count <= max_retries:
                try:
                    response = client.get(base_url, params=params)
                    response.raise_for_status()  # Check for HTTP errors

                    data = response.json()
                    for trade in data["results"]:
                        yield trade

                    next_url = data.get("next_url")
                    if not next_url:
                        break

                    base_url = next_url
                    params = {"apiKey": api_key}
                    break  # Successful request, exit retry loop

                except (HTTPStatusError, RequestError) as e:
                    print(f"Request failed: {e}, attempt {retry_count + 1}")
                    retry_count += 1
                    time.sleep(
                        initial_backoff * (2**retry_count)
                    )  # Exponential backoff

                except Exception as e:
                    print(f"An unexpected error occurred: {e}")
                    break  # Non-retriable error, break the loop

            if retry_count > max_retries:
                print("Max retries exceeded. Exiting...")
                break  # Max retries exceeded, exit the main loop

    finally:
        client.close()


if __name__ == "__main__":
    last_print_time = 0  # Initialize the last print time to zero

    try:
        for trade in polygon_trades_streamer(
            ticker="X:BTC-USD",
            timestamp_gte="2016-01-01",
            timestamp_lte="2024-04-30",
            limit=50000,
            sort="timestamp",
            order="asc",
            api_key=api_key,
        ):
            current_time = time.time()
            if (
                current_time - last_print_time >= 5
            ):  # Check if at least 5 seconds have passed
                print(
                    f"rlast timestamp: {convert_timestamp(trade['participant_timestamp'])}",
                    end="",
                    flush=True,
                )
                last_print_time = current_time  # Update the last print time

    except Exception as e:
        print(f"An error occurred: {e}")

The fault handler log:

Fatal Python error: Segmentation fault

Current thread 0x0000727a16279740 (most recent call first):
  File "/home/furkan/Code/Deep-Learning-Prediction-Model/data/api/polygon/trade_generator.py", line 60 in polygon_trades_streamer
  File "/home/furkan/Code/Deep-Learning-Prediction-Model/data/api/polygon/trade_generator.py", line 95 in <module>

Extension modules: _brotli (total: 1)