Question

I have to write a small app which downloads a few thousand files. Some of these files contain reference to other files that must be downloaded as part of the same process. The following code downloads the initial list of files, but I would like to download the others files as part of the same loop. What is happening here is that the loop completes before the first request come back. Any idea how to achieve this?

var countdownLatch = new CountdownEvent(Urls.Count);

string url;
while (Urls.TryDequeue(out url))
{
    HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
    webRequest.BeginGetResponse(
        new AsyncCallback(ar =>
        {
            using (HttpWebResponse response = (ar.AsyncState as HttpWebRequest).EndGetResponse(ar) as HttpWebResponse)
            {
                using (var sr = new StreamReader(response.GetResponseStream()))
                {
                    string myFile = sr.ReadToEnd();

                    // TODO: Look for a reference to another file. If found, queue a new Url.
                }
            }
        }), webRequest);
}

ce.Wait();

Answer 1

One solution which comes to mind is to keep track of the number of pending requests and only finish the loop once no requests are pending and the Url queue is empty:

string url;
int requestCounter = 0;
int temp;
AutoResetEvent requestFinished = new AutoResetEvent(false);
while (Interlocked.Exchange(requestCounter, temp) > 0 || Urls.TryDequeue(out url))
{
    if (url != null)
    {
        Interlocked.Increment(requestCounter);
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
        webRequest.BeginGetResponse(
            new AsyncCallback(ar =>
            {
                try {
                    using (HttpWebResponse response = (ar.AsyncState as HttpWebRequest).EndGetResponse(ar) as HttpWebResponse)
                    {
                        using (var sr = new StreamReader(response.GetResponseStream()))
                        {
                            string myFile = sr.ReadToEnd();

                            // TODO: Look for a reference to another file. If found, queue a new Url.
                        }
                    }
                } 
                finally { 
                    Interlocked.Decrement(requestCounter); 
                    requestFinished.Set();
                }
            }), webRequest);
    }
    else
    {
        // no url but requests are still pending
        requestFinished.WaitOne();
    }
}