C#•5mo ago

✅ Using IProgress with await Task.WhenAll(tasks);

I am trying to introduce a IProgress object into the code below. Can't for the life of me figure how to do it. I need the code this way (rather than via a foreach loop) because I am querying an API which limits the number of concurrent threads. The IProgress objects main ProgressEventArgs will be how many items have been processed and also the url used in each task instance. Thus I need some way to keep track of how many taks have been completed. Any steers? The code:

    
public class Scraper
{
    
    private SemaphoreSlim semaphoreSlim = 7;
    public void Scraper(int concurrentThreads)
    {
        semaphoreSlim = new SemaphoreSlim(concurrentThreads);
        ServicePointManager.FindServicePoint(new Uri(baseUrl)).ConnectionLimit = concurrentThreads;
    }

    public async Task<List<ApiGetStringOutcome>> GetStringsFromUrlList(List<string> apiUrls)
    {
        List<Task<ApiGetStringOutcome>> tasks = new List<Task<ApiGetStringOutcome>>();

        foreach (string apiUrl in apiUrls)
        {
            tasks.Add(GetStringDataAsync(apiUrl));
        }

        await Task.WhenAll(tasks);

        return new List<ApiGetStringOutcome>(tasks.Select(t => t.Result));
    }

    private async Task<ApiGetStringOutcome> GetStringDataAsync(string apiUrl, ApiGameSearchParameters gameSearchParameters = null)
    {
        await semaphoreSlim.WaitAsync(); // Wait until semaphore is available

        try
        {
            HttpResponseMessage response = await httpClient.GetAsync(apiUrl);
        }
        catch (Exception ex)
        {
            // Stuff here
        }
        finally
        {
            semaphoreSlim.Release();
        }
    }
}

public class MainApp
{
    public void Start()
    {        
        // blah...
        getStringOutcomes = await Task.Run(() => apiDataService.GetStringsFromUrlList(urlList));
    }
    
}

    
public class Scraper
{
    
    private SemaphoreSlim semaphoreSlim = 7;
    public void Scraper(int concurrentThreads)
    {
        semaphoreSlim = new SemaphoreSlim(concurrentThreads);
        ServicePointManager.FindServicePoint(new Uri(baseUrl)).ConnectionLimit = concurrentThreads;
    }

    public async Task<List<ApiGetStringOutcome>> GetStringsFromUrlList(List<string> apiUrls)
    {
        List<Task<ApiGetStringOutcome>> tasks = new List<Task<ApiGetStringOutcome>>();

        foreach (string apiUrl in apiUrls)
        {
            tasks.Add(GetStringDataAsync(apiUrl));
        }

        await Task.WhenAll(tasks);

        return new List<ApiGetStringOutcome>(tasks.Select(t => t.Result));
    }

    private async Task<ApiGetStringOutcome> GetStringDataAsync(string apiUrl, ApiGameSearchParameters gameSearchParameters = null)
    {
        await semaphoreSlim.WaitAsync(); // Wait until semaphore is available

        try
        {
            HttpResponseMessage response = await httpClient.GetAsync(apiUrl);
        }
        catch (Exception ex)
        {
            // Stuff here
        }
        finally
        {
            semaphoreSlim.Release();
        }
    }
}

public class MainApp
{
    public void Start()
    {        
        // blah...
        getStringOutcomes = await Task.Run(() => apiDataService.GetStringsFromUrlList(urlList));
    }
    
}

3 Replies

not guilty•5mo ago

i don't know what progress you have to track, but i would say usually you make your class that that implements IProject that keesp the context and bridges the internal call to the event the progress should call then you pass your class inside your methods down until where it's necessary and since these are all done by you there should be no barrier to do so

stigzler•5mo ago

🤦 Coming back to it with fresh eyes - it was simple really:

public class ApiQueryProgressEventArgs
{
    public int TotalObjectsToProcess { get; set; } = 0;
    public int TotalObjectsProcessed { get; set; } = 0;
}


public class Scraper
{

    public IProgress<ApiQueryProgressEventArgs> QueryProgress { get; set; }
    public ApiQueryProgressEventArgs QueryProgressEventArgs {  get; set; }
    
    public async Task<List<ApiGetStringOutcome>> GetStringsFromUrlList(List<string> apiUrls)
    {
        QueryProgressEventArgs.TotalObjectsToProcess = apiUrls.Count();
        // rest as per OP        
    }
        
    private async Task<ApiGetStringOutcome> GetStringDataAsync(string apiUrl, ApiGameSearchParameters gameSearchParameters = null)
    {
        // rest as per OP
        finally
        {
            QueryProgressEventArgs.TotalObjectsProcessed += 1;
            QueryProgress.Report(QueryProgressEventArgs);
            semaphoreSlim.Release(); 
        }        
    }
}

public class ApiQueryProgressEventArgs
{
    public int TotalObjectsToProcess { get; set; } = 0;
    public int TotalObjectsProcessed { get; set; } = 0;
}


public class Scraper
{

    public IProgress<ApiQueryProgressEventArgs> QueryProgress { get; set; }
    public ApiQueryProgressEventArgs QueryProgressEventArgs {  get; set; }
    
    public async Task<List<ApiGetStringOutcome>> GetStringsFromUrlList(List<string> apiUrls)
    {
        QueryProgressEventArgs.TotalObjectsToProcess = apiUrls.Count();
        // rest as per OP        
    }
        
    private async Task<ApiGetStringOutcome> GetStringDataAsync(string apiUrl, ApiGameSearchParameters gameSearchParameters = null)
    {
        // rest as per OP
        finally
        {
            QueryProgressEventArgs.TotalObjectsProcessed += 1;
            QueryProgress.Report(QueryProgressEventArgs);
            semaphoreSlim.Release(); 
        }        
    }
}

Unknown User•5mo ago

Message Not Public

Gaming

Programming

✅ Using IProgress with await Task.WhenAll(tasks);