Question

I’m testing a self written element generator (ICollection<string>) and compare the calculated count to the actual count to get an idea if there’s an error or not in my algorithm.

As this generator can generate lots of elements on demand I’m looking in Partitioner<string> and I have implemented a basic one which seems to also produce valid enumerators which together give the same amount of strings as calculated.

Now I want to test how this behaves if run parallel (again first testing for correct count):

MyGenerator generator = new MyGenerator();
MyPartitioner partitioner = new MyPartitioner(generator);

int isCount = partitioner.AsParallel().Count();
int shouldCount = generator.Count;

bool same = isCount == shouldCount; // false

I don’t get why this count is not equal! What is the ParallelQuery<string> doing?

generator.Count() == generator.Count // true

partitioner.GetPartitions(xyz).Select(enumerator =>
    {
        int count = 0;
        while (enumerator.MoveNext())
        {
            count++;
        }
        return count;
    }).Sum() == generator.Count // true

So, I’m currently not seeing an error in my code. Next I tried to manualy count that ParallelQuery<string>:

int count = 0;
partitioner.AsParallel().ForAll(e => Interlocked.Increment(ref count));
count == generator.Count // true

Summed up: Everyone counts my enumerable correct, ParallelQuery.ForAll enumerates exactly generator.Count elements. But what does ParallelQuery.Count()?

If the correct count is something about 10k, ParallelQuery sees 40k.

---

    internal sealed class PartialWordEnumerator : IEnumerator<string>
    {
        private object sync = new object();

        private readonly IEnumerable<char> characters;

        private readonly char[] limit;

        private char[] buffer;
        private IEnumerator<char>[] enumerators;

        private int position = 0;

        internal PartialWordEnumerator(IEnumerable<char> characters, char[] state, char[] limit)
        {
            this.characters = new List<char>(characters);

            this.buffer = (char[])state.Clone();

            if (limit != null)
            {
                this.limit = (char[])limit.Clone();
            }

            this.enumerators = new IEnumerator<char>[this.buffer.Length];

            for (int i = 0; i < this.buffer.Length; i++)
            {
                this.enumerators[i] = SkipTo(state[i]);
            }
        }

        private IEnumerator<char> SkipTo(char c)
        {
            IEnumerator<char> first = this.characters.GetEnumerator();
            IEnumerator<char> second = this.characters.GetEnumerator();

            while (second.MoveNext())
            {
                if (second.Current == c)
                {
                    return first;
                }

                first.MoveNext();
            }

            throw new InvalidOperationException();
        }

        private bool ReachedLimit
        {
            get
            {
                if (this.limit == null)
                {
                    return false;
                }

                for (int i = 0; i < this.buffer.Length; i++)
                {
                    if (this.buffer[i] != this.limit[i])
                    {
                        return false;
                    }
                }

                return true;
            }
        }

        public string Current
        {
            get
            {
                if (this.buffer == null)
                {
                    throw new ObjectDisposedException(typeof(PartialWordEnumerator).FullName);
                }

                return new string(this.buffer);
            }
        }

        object IEnumerator.Current
        {
            get { return this.Current; }
        }

        public bool MoveNext()
        {
            lock (this.sync)
            {
                if (this.position == this.buffer.Length)
                {
                    this.position--;
                }

                if (this.position == -1)
                {
                    return false;
                }

                IEnumerator<char> enumerator = this.enumerators[this.position];

                if (enumerator.MoveNext())
                {
                    this.buffer[this.position] = enumerator.Current;
                    this.position++;

                    if (this.position == this.buffer.Length)
                    {
                        return !this.ReachedLimit;
                    }
                    else
                    {
                        return this.MoveNext();
                    }
                }
                else
                {
                    this.enumerators[this.position] = this.characters.GetEnumerator();
                    this.position--;

                    return this.MoveNext();
                }
            }
        }

        public void Dispose()
        {
            this.position = -1;
            this.buffer = null;
        }

        public void Reset()
        {
            throw new NotSupportedException();
        }
    }

---

    public override IList<IEnumerator<string>> GetPartitions(int partitionCount)
    {
        IEnumerator<string>[] enumerators = new IEnumerator<string>[partitionCount];

        List<char> characters = new List<char>(this.generator.Characters);

        int length = this.generator.Length;

        int characterCount = this.generator.Characters.Count;

        int steps = Math.Min(characterCount, partitionCount);

        int skip = characterCount / steps;

        for (int i = 0; i < steps; i++)
        {
            char c = characters[i * skip];

            char[] state = new string(c, length).ToCharArray();
            char[] limit = null;

            if ((i + 1) * skip < characterCount)
            {
                c = characters[(i + 1) * skip];
                limit = new string(c, length).ToCharArray();
            }

            if (i == steps - 1)
            {
                limit = null;
            }

            enumerators[i] = new PartialWordEnumerator(characters, state, limit);
        }

        for (int i = steps; i < partitionCount; i++)
        {
            enumerators[i] = Enumerable.Empty<string>().GetEnumerator();
        }

        return enumerators;
    }

Answer 1

EDIT: I believe I have found the solution. According to the documentation on IEnumerable.MoveNext (emphasis mine):

If MoveNext passes the end of the collection, the enumerator is
positioned after the last element in the collection and MoveNext
returns false. When the enumerator is at this position, subsequent
calls to MoveNext also return false until Reset is called.

According to the following logic:

    private bool ReachedLimit
    {
        get
        {
            if (this.limit == null)
            {
                return false;
            }

            for (int i = 0; i < this.buffer.Length; i++)
            {
                if (this.buffer[i] != this.limit[i])
                {
                    return false;
                }
            }

            return true;
        }
    }

The call to MoveNext() will return false only one time – when the buffer is exactly equal to the limit. Once you have passed the limit, the return value from ReachedLimit will start to become false again, making return !this.ReachedLimit return true, so the enumerator will continue past the end of the limit all the way until it runs out of characters to enumerate. Apparently, in the implementation of ParallelQuery.Count(), MoveNext() is called multiple times when it has reached the end, and since it starts to return a true value again, the enumerator happily continues returning more elements (this is not the case in your custom code that walks the enumerator manually, and apparently also is not the case for the ForAll call, so they “accidentally” return the correct results).

The simplest fix to this is to remember the return value from MoveNext() once it becomes false:

private bool _canMoveNext = true;
public bool MoveNext()
{
    if (!_canMoveNext) return false;
    ...

        if (this.position == this.buffer.Length)
        {
            if (this.ReachedLimit) _canMoveNext = false;
    ...
}

Now once it begins returning false, it will return false for every future call and this returns the correct result from AsParallel().Count(). Hope this helps!

---

The documentation on Partitioner notes (emphasis mine):

The static methods on Partitioner are all thread-safe and may
be used concurrently from multiple threads. However, while a created
partitioner is in use, the underlying data source should not be
modified, whether from the same thread that is using a partitioner or
from a separate thread.

From what I can understand of the code you have given, it would seem that ParallelQuery.Count() is most likely to have thread-safety issues because it may possibly be iterating multiple enumerators at the same time, whereas all the other solutions would require the enumerators to be run synchronized. Without seeing the code you are using for MyGenerator and MyPartitioner is it difficult to determine if thread-safety issues could be the culprit.

---

To demonstrate, I have written a simple enumerator that returns the first hundred numbers as strings. Also, I have a partitioner, that distributes the elements in the underlying enumerator over a collection of numPartitions separate lists. Using all the methods you described above on our 12-core server (when I output numPartitions, it uses 12 by default on this machine), I get the expected result of 100 (this is LINQPad-ready code):

void Main()
{
    var partitioner = new SimplePartitioner(GetEnumerator());

    GetEnumerator().Count().Dump();

    partitioner.GetPartitions(10).Select(enumerator =>
    {
        int count = 0;
        while (enumerator.MoveNext())
        {
            count++;
        }
        return count;
    }).Sum().Dump();

    var theCount = 0;
    partitioner.AsParallel().ForAll(e => Interlocked.Increment(ref theCount));
    theCount.Dump();

    partitioner.AsParallel().Count().Dump();
}

// Define other methods and classes here
public IEnumerable<string> GetEnumerator()
{
    for (var i = 1; i <= 100; i++)
        yield return i.ToString();
}

public class SimplePartitioner : Partitioner<string>
{
    private IEnumerable<string> input;
    public SimplePartitioner(IEnumerable<string> input)
    {
        this.input = input;
    }

    public override IList<IEnumerator<string>> GetPartitions(int numPartitions)
    {
        var list = new List<string>[numPartitions];
        for (var i = 0; i < numPartitions; i++)
            list[i] = new List<string>();
        var index = 0;
        foreach (var s in input)
            list[(index = (index + 1) % numPartitions)].Add(s);

        IList<IEnumerator<string>> result = new List<IEnumerator<string>>();
        foreach (var l in list)
            result.Add(l.GetEnumerator());
        return result;
    }
}

Output:

100
100
100
100

This clearly works. Without more information it is impossible to tell you what is not working in your particular implementation.