Question

I am uploading large files via Silverlight and have implemented a chunking function. It works fine, however, if I upload three large (500mb) files in a row I still get out of memory exceptions. Below is my code, can you spot anything that is missing?

const int _ReadSize = 2097152;

byte[] _Buffer = new byte[_ReadSize];

    /// <summary>
    /// This method will do the initial read and write and send off the first chunk of data.
    /// This is where the filestream is opened from the file info.
    /// It also passes a set of parameters to the next call. These are:
    /// * bytesRead - the number of bytes that was actually read from the file with stream.Read
    /// * stream - This is the filestream
    /// * offset - This is the updated offset that has been moved to the position in the stream we are currently at
    /// </summary>
    void DoWork()
    {
        FileStream stream = _SelectedFile.OpenRead();

        ServerAvailable = false;

        bool startRead = true;

        int bytesRead = 0;

        bytesRead = stream.Read(_Buffer, 0, _ReadSize);

        int offset = bytesRead;

        List<object> args = new List<object>();
        args.Add(bytesRead);
        args.Add(stream);
        args.Add(offset);

        IDataManipulationService client = new DataManipulationServiceClient();
        client.BeginUploadLargeFile(_Buffer, (int)_SelectedFile.Length, FileName, startRead, offset - bytesRead, bytesRead, FinishedUploadPiece, args);
    }

    /// <summary>
    /// This method is called once the previous call to the web server has been completed.
    /// It will read the next chunk of the file and send that through to the web server next.
    /// If 0 bytes were read from the previous read on the stream; it will do the following:
    /// - Close the file stream
    /// - Dispose the file stream
    /// - set the FileInfo to null
    /// - Reset the FileSize, UploadProgress and FileName variables to default values
    /// - Make the buttons available for use
    /// </summary>
    /// <param name="result">The result contains the information about the outcome of the previous call. This also contains the args parameter sent through with the previous call.</param>
    void FinishedUploadPiece(IAsyncResult result)
    {
        if (result.IsCompleted)
        {
            List<object> args = (List<object>)result.AsyncState;

            int bytesRead = (int)args[0];
            FileStream stream = (FileStream)args[1];
            int offset = (int)args[2];

            if (bytesRead != 0)
            {
                UploadProgress += bytesRead;

                if (UploadProgress == FileSize)
                {
                    FileSize = 0;
                    UploadProgress = 0;
                    FileName = String.Empty;

                    ServerAvailable = true;
                    stream.Close();
                    stream.Dispose();
                    _SelectedFile = null;
                }
                else
                {
                    bytesRead = stream.Read(_Buffer, 0, _ReadSize);

                    offset += bytesRead;

                    args = new List<object>();

                    args.Add(bytesRead);
                    args.Add(stream);
                    args.Add(offset);

                    IDataManipulationService client = new DataManipulationServiceClient();
                    client.BeginUploadLargeFile(_Buffer, (int)_SelectedFile.Length, FileName, false, offset - bytesRead, bytesRead, FinishedUploadPiece, args);
                }
            }
        }
    }

To clarify some stuff:
_SelectedFile is of type FileInfo and I create a new connection to the server each time I want to send data through but have also tried having a global injected connection.

Answer 1

I would try to refactor the code to use Error Codes and not pass the stream around.

void DoWorkAsync()
{
    IDataManipulationService client = new DataManipulationServiceClient();

    try
    {
        using(FileStream stream = _SelectedFile.OpenRead())
        {
            int streamLength = (int)_SelectedFile.Length
            ServerAvailable = false;

            bool startRead = true;

            int bytesRead;
            int offset = 0;

            while((bytesRead = stream.Read(_Buffer, 0, _ReadSize)) > 0)
            {
                offset += bytesRead;
                UploadState state = await client.UploadChunk(_Buffer, streamLength, FileName, offset, bytesRead)

                if(state == UploadState.Error)
                    //error Handling

                if(state == UploadState.Corrupt)
                    //Retry send

                if(state == UploadState.Success)
                    continue;
            }
        }
    }
    catch(Exception e) {} //TODO: Replace Exception with some meaningful exception.
    finally
    {
        ServerAvailable = true;
    }
}

enum UploadState
{
    Success,
    Error,
    Corrupt
}

Alternatives to async/await:

See What should I do to use Task<T> in .NET 2.0? for a a lightweight implementation of async/await.

Reactive Extensions will provide an implementation of System.Threading.dll which contains Task or you can use NuGet.

Here is an example implementation:

void DoWork()
{
    IDataManipulationService client = new DataManipulationServiceClient();

    try
    {
        using(FileStream stream = _SelectedFile.OpenRead())
        {
            int streamLength = (int)_SelectedFile.Length
            ServerAvailable = false;

            bool startRead = true;

            int bytesRead;
            int offset = 0;

            while((bytesRead = stream.Read(_Buffer, 0, _ReadSize)) > 0)
            {
                offset += bytesRead;
                Task<UploadState> task = client.UploadChunk(_Buffer, streamLength, FileName, offset, bytesRead);

                while(true)
                {
                    if(task.IsCompleted)
                    {
                        UploadState state = task.Result;
                        break;
                    }

                    if(task.IsFaulted)
                    {
                        var exception = task.Exception;
                        //Deal with errors
                        break;
                    }
                }

                if(state == UploadState.Error)
                    //error Handling

                if(state == UploadState.Corrupt)
                    //Retry send

                if(state == UploadState.Success)
                    continue;
            }
        }
    }
    catch(Exception e) {} //TODO: Replace Exception with some meaningful exception.
    finally
    {
        ServerAvailable = true;
    }
}

enum UploadState
{
    Success,
    Error,
    Corrupt
}