Question

I am trying to run SharePoint’s SPSiteDataQuery for all web in the Site Collection for a collection of lists. The average time to get data from all sites for one list [spWeb.GetSiteData(dataQuery)] is anywhere between 1 to 20 seconds. The list collection can virtually have any number of lists in it.

I have implemented the threading logic as below. But I get different amount of records on every call. I am not much experienced with multi-threading architecture. What is going on, and is it worth my thinking about a better threading architecture?

private static Dictionary<string,string> _workTypes = new Dictionary<string, string>();
private static Dictionary<string,string> _workspaces = new Dictionary<string, string>();
private static List<string> _processedMyWorkItems = new List<string>(); 
private static readonly object Locker = new object();

---

var threads = new List<Thread>();
var processedMyWorkData = new Dictionary<string, List<XElement>>();

foreach (string selectedList in selectedLists.Distinct().OrderBy(l => l))
{
    string theSelectedList = selectedList;

    string listIds = string.Empty;

    foreach (string listId in GetListIdsFromDb(selectedList, spWeb, archivedWebs)
        .Where(listId => !selectedListIds.Contains(listId)))
    {
        selectedListIds.Add(listId);
        listIds += string.Format(@"<List ID='{0}'/>", listId);
    }

    if (string.IsNullOrEmpty(listIds)) continue;

    dataQuery.Lists = string.Format("<Lists MaxListLimit='0'>{0}</Lists>", listIds);

    var thread = new Thread(() =>
    {
        List<XElement> processedMyWork = ProcessMyWork(selectedFields, spSite, spWeb, dataQuery, fieldTypes);

        lock (Locker)
        {
            processedMyWorkData.Add(theSelectedList, processedMyWork);
        }
    }) { Name = theSelectedList };

    threads.Add(thread);
    thread.Start();
}

bool done = false;

while (!done)
{
    Thread.Sleep(500);

    bool noThreadIsAlive = true;
    var threadsCompleted = new List<Thread>();

    foreach (var thread in threads)
    {
        if (thread.IsAlive)
        {
            noThreadIsAlive = false;
        }
        else
        {
            threadsCompleted.Add(thread);
        }
    }

    foreach (var thread in threadsCompleted)
    {
        threads.Remove(thread);

        string key = thread.Name;
        foreach (var xElement in processedMyWorkData[key])
        {
            result.Element("MyWork").Add(xElement);
        }

        lock (Locker)
        {
            processedMyWorkData.Remove(key);
        }
    }

    done = noThreadIsAlive;
}

Answer 1

Just looking at the synchronization issue, you need to be using the WaitAll() rather than looping around WaitOne() or polling on the IsAlive property.
To set it up, you’ll need an event for each thread then add them to an array in the main thread that will uses it to wait on all the events at once.
Note that there is a maximum of 64 events that can be monitored by WaitAll().

http://msdn.microsoft.com/en-us/library/3dasc8as(v=vs.80).aspx