Question

I am trying to create a class to send multiple http request simultaneously on different threads to try to speed up getting multiple documents over the internet.

I implemented this, but when I increase from 1 to 2 threads, execution time doubles, going from 1 to 4 threads, execution time quadruples. I would think it should be faster on more than 1 thread!

Here is the code, maybe I have some wierd issue.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.ComponentModel;

namespace scomA3proj
{
    public class MultithreadedHttpRequests_James : IMultithreadedHttpRequests_James
    {
        List<string> Urls;
        string[] responses;
        public List<string> getHttpResponses(List<string> urls, int numThreads)
        {
            this.Urls = urls;
            responses = new string[urls.Count];
            List<Thread> threads = new List<Thread>();
            for (int i = 0; i < numThreads; i++)
            {
                Thread bgw = new Thread(new ParameterizedThreadStart(bgw_DoWork));
                bgw.Start();
                threads.Add(bgw);
            }
            for (int i = 0; i < numThreads; i++)
            {
                threads[i].Join();
            }
            return responses.ToList();
        }

        void bgw_DoWork(object sender)
        {
            while (true)
            {
                int index = getNext();
                if (index == -1) break;
                string s = Urls[index];
                responses[index] = HttpRequestWrapper.getResponse(s);
            }
        }
        int counter = 0;
        int getNext()
        {
            int res = 0;
            lock (this)
            {
                res = counter;
                counter++;
            }
            if (res >= Urls.Count) 
                return -1;

            return res;
        }
    }
}


using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;

namespace scomA3proj
{
    public sealed class HttpRequestWrapper
    {
        /// <summary>
        /// Gets the HTTP response from a web page. Headers are used based on the useHeaders flag.
        /// </summary>
        /// <param name="url">URI formatted URL(example:"http://www.yahoo.com").</param>
        /// <returns>Returns Html source of requested page.</returns>
        public static string getResponse(string url)
        {
            string result = "";
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                StreamReader reader = new StreamReader(response.GetResponseStream());
                result = reader.ReadToEnd();
            }
            catch (Exception ex)
            {
                result = "error";
            }
            return result;
        }
    }
}

Update:
I added this, but it did not help

ServicePointManager.DefaultConnectionLimit = numThreads;

I think maybe the network path I am connected to (university) is limiting to one outbound connection per pc or something and penalizing me for requesting multiple…idk…

Answer 1

A couple of things could be going on here, the first is that you’re reaching the connection count limit allowed for web requests. By default this is set to 2, so to change that, at the beginning of the function, you can update that to the number of threads you’re using.

ServicePointManager.DefaultConnectionLimit = 10; 
// 10 should match the number of threads you're executing

Other than that, it depends on where you’re monitoring the execution time. If you’re monitoring the overall execution time, you should watch out for the miscellaneous stuff you’re doing, like implementing a Queue that you’re not using.