Question

List is a wrapper over array. While you add items to the list it creates bigger and bigger array undercover (and previous array is garbage collected). But if you treat large lists at some point you will get OutOfMemoryException even if there is free memory due to memory fragmentation. I am looking for an ICollection implementation which would work with set of arrays undercover similar to what MemoryTributary does.

Update.

I have found BigArray implementation here:

http://blogs.msdn.com/b/joshwil/archive/2005/08/10/450202.aspx.

While it tries to solve other problem (creating an array of >2GB size), it solves my problem too. But this implementation is not full and even does not compile. So if I don’t find any better, I will improve this one and use it.

Answer 1

I haven’t found any good implementation. So I have written my own ChunkyList. Generally ChunkyList is a List of arrays wrapper. Initially block has size of 1 but it is multiplied by two (when it reaches MaxBlockSize, a next block is created) every time you need to extend the current block (the behavior similar to the List).

Here is a generic ChunkyList:

public class ChunkyList<T> : IList<T>
{
    public ChunkyList()
    {
        MaxBlockSize = 65536;
    }

    public ChunkyList(int maxBlockSize)
    {
        MaxBlockSize = maxBlockSize;
    }

    private List<T[]> _blocks = new List<T[]>();

    public int Count { get; private set; }

    public int MaxBlockSize { get; private set; }

    public bool IsReadOnly
    {
        get { throw new NotImplementedException(); }
    }

    public IEnumerator<T> GetEnumerator()
    {
        var index = 0;
        foreach (var items in _blocks)
            foreach (var item in items)
            {
                yield return item;
                if (Count <= ++index)
                    break;
            }
    }

    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }

    public void Add(T item)
    {
        var indexInsideBlock = GetIndexInsideBlock(Count);
        if (indexInsideBlock == 0)
            _blocks.Add(new T[1]);
        else
        {
            var lastBlockIndex = _blocks.Count - 1;
            var lastBlock = _blocks[lastBlockIndex];
            if(indexInsideBlock >= lastBlock.Length)
            {
                var newBlockSize = lastBlock.Length*2;
                if (newBlockSize >= MaxBlockSize)
                    newBlockSize = MaxBlockSize;

                _blocks[lastBlockIndex] = new T[newBlockSize];
                Array.Copy(lastBlock, _blocks[lastBlockIndex], lastBlock.Length);
            }
        }

        _blocks[GetBlockIndex(Count)][indexInsideBlock] = item;

        Count++;
    }

    public void AddRange(IEnumerable<T> items)
    {
        foreach (var item in items)
            Add(item);
    }

    public void Clear()
    {
        throw new NotImplementedException();
    }

    public bool Contains(T item)
    {
        throw new NotImplementedException();
    }

    public void CopyTo(T[] array, int arrayIndex)
    {
        throw new NotImplementedException();
    }

    public bool Remove(T item)
    {
        throw new NotImplementedException();
    }

    public int IndexOf(T item)
    {
        throw new NotImplementedException();
    }

    public void Insert(int index, T item)
    {
        throw new NotImplementedException();
    }

    public void RemoveAt(int index)
    {
        throw new NotImplementedException();
    }

    public T this[int index]
    {
        get
        {
            if (index >= Count)
                throw new ArgumentOutOfRangeException("index");

            var blockIndex = GetBlockIndex(index);
            var block = _blocks[blockIndex];

            return block[GetIndexInsideBlock(index)];
        }
        set { throw new NotImplementedException(); }
    }

    private int GetBlockIndex(int index)
    {
        return index / MaxBlockSize;
    }

    private long GetIndexInsideBlock(int index)
    {
        return index % MaxBlockSize;
    }
}

And here are tests which prove this implementation works:

 [TestClass]
    public class ChunkyListTests
    {
        [TestMethod]
         public void GetEnumerator_NoItems()
         {
             var chunkyList = new ChunkyList<float>();

             var wasInsideForeach = false;
             foreach (var item in chunkyList)
                 wasInsideForeach = true;

             Assert.IsFalse(wasInsideForeach);
         }

        [TestMethod]
        public void GetEnumerator_MaxBlockSizeOfThreeWithThreeItems()
        {
            var chunkyList = new ChunkyList<float> (3) { 1, 2, 3 };

            var wasInsideForeach = false;
            var iteratedItems = new List<float>();
            foreach (var item in chunkyList)
            {
                wasInsideForeach = true;
                iteratedItems.Add(item);
            }

            Assert.IsTrue(wasInsideForeach);
            CollectionAssert.AreEqual(new List<float> { 1, 2, 3 }, iteratedItems);
        }

        [TestMethod]
        public void GetEnumerator_MaxBlockSizeOfTwoWithThreeItems()
        {
            var chunkyList = new ChunkyList<float>(2) {1,2,3};
            var wasInsideForeach = false;
            var iteratedItems = new List<float>();

            foreach (var item in chunkyList)
            {
                wasInsideForeach = true;
                iteratedItems.Add(item);
            }

            Assert.IsTrue(wasInsideForeach);
            CollectionAssert.AreEqual(new List<float>() { 1, 2, 3 }, iteratedItems);
            Assert.AreEqual(chunkyList.MaxBlockSize, 2);
        }
    }

P. S. I have implemented only those IList methods which are used in my code. So you are welcome to improve this implementation.