Looking to figure out where exactly I’m going wrong.
I have a program that queries an API, the API returns data in chunks and will return a response header with a URL to the next chunk of data, if there is more.
I am trying to call the query the API, when I get a response, I would like to start processing that data as I’m waiting for the next chunk of data to arrive. But it seems like I’m waiting for the data, processing it and then waiting/requesting the next bunch
JArray jsonObjects = new JArray();
Task.Run(async () =>
{
jsonObjects = await GetObjectsJSONAsync();
}).GetAwaiter().GetResult();
private async Task<JArray> GetObjectsJSONAsync()
{
/*
* HttpClient setup here
*/
var response = "";
var headers = "";
var total = 0;
JArray productsFromURL = new JArray();
HttpResponseMessage result = await client.GetAsync("/products/skus/");
if (result.IsSuccessStatusCode)
{
headers = result.Headers.GetValues("Link").FirstOrDefault();
response = await result.Content.ReadAsStringAsync();
jsonObjectsFromURL = JArray.Parse(response);
ProcessObjects(productsFromURL);
while (headers.Contains("rel="next"")) //while header contains next link
{
var url = headers.Split(';')[0].Replace("<", "").Replace(">", "");
result = await client.GetAsync(url);
try
{
response = await result.Content.ReadAsStringAsync();
jsonObjectsFromURL = JArray.Parse(response);
ProcessObjects(jsonObjectsFromURL); //adds processed objects to a ConcurrentBag
headers = result.Headers.GetValues("Link").FirstOrDefault();
}
catch (Exception e)
{
//This means that there are no more pages to get
headers = "";
}
}
}
return jsonObjectsFromURL; //dont care about the return here
}