I know there are many questions on this topic and I tried all the possible solutions but still not working. Please see my problem if you can help, much appreciated.

I have created a console application that will be responsible for downloading mp4 files when as soon as they are available in the DB. So it means, I have to keep my thread running in the console application and keep checking the database every 5 minutes to see if data is available, then start downloading with multiple threads.

The list of problems I am facing here is:

1. When the thread starts again after 5 minutes of sleep, it processes the same records again whereas the same download might be running. Is there any way to sleep the thread for the time until all downloads get completed?

2. To solve the above problem, I kept a flag in DB to not process the same record again but still when the second time thread starts for newer records, it stuck the download after some interval and hangs the process.

I need help if my code is correct or not, can I get time to sleep my thread until all 5 downloads get completed, OR why is my thread stuck after 2nd trial?

Also, I encountered an error in the main Async download function as below when it stuck or hung the download, “unable to read data from the transport connection”

static void Main(string[] args)
{
    try
    {
        ServicePointManager.DefaultConnectionLimit = 1000;
        VideoDownloadProcessor videoDownloadProcessor = new VideoDownloadProcessor();
        videoDownloadProcessor.StartProcess();
    }
    catch (Exception ex)
    {
        // Handle error
    }
}

public class VideoToAudioProcessor
{   
    public async void StartProcess()
    {
        try
        {
            while (true)
            {
                List<Task> downloadTasks = new List<Task>();
                // Fetch records from DB in dbRecords;
                foreach (var mp4s in dbRecords)
                {
                    downloadTasks.Add(this.DownloadData(mp4Url));
                }
                
                // Wait for all download tasks to complete
                await Task.WhenAll(downloadTasks);
                
                Thread.Sleep(_sleepMinutes); // Thread will sleep for defined minutes and start again to check the DB to process records if any
            }
        }
        catch (Exception ex)
        {
            // Handle error
        }
    }
    
     private async void DownloadData(string mp4Url)
     {
        string resultDownload = await DownloadFileAsync(mp4Url, videofileStoragePath);
        
        if (resultDownload == "success")
        {
            // Update in DB that download of that file is completed.
        }
     }
     
    static async Task<string> DownloadFileAsync(string fileUrl, string outputPath)
    {
        string result = string.Empty;
        try
        {
            using (HttpClient client = new HttpClient())
            {
                client.Timeout = TimeSpan.FromHours(5);

                using (HttpResponseMessage response = await client.GetAsync(fileUrl, HttpCompletionOption.ResponseHeadersRead))
                {
                    response.EnsureSuccessStatusCode();

                    using (Stream contentStream = await response.Content.ReadAsStreamAsync(), fileStream = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true))
                    {
                        byte[] buffer = new byte[8192];
                        int bytesRead;
                        long totalBytesRead = 0L;
                        long totalBytes = response.Content.Headers.ContentLength ?? -1L;

                        Console.WriteLine($"Downloading {Path.GetFileName(fileUrl)}...");

                        while ((bytesRead = await contentStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
                        {
                            await fileStream.WriteAsync(buffer, 0, bytesRead);
                            totalBytesRead += bytesRead;

                            if (totalBytes > 0)
                            {
                                double percentage = ((double)totalBytesRead / totalBytes) * 100;
                                Console.WriteLine($"Progress: {percentage:F2}%");
                            }
                        }
                    }
                }
            }

            result = "success";
        }
        catch (Exception ex)
        {
            result = ex.Message + " " + ex.StackTrace;
            Console.WriteLine(ex.Message + " " + ex.StackTrace);
        }

        return result;
    }
}