Question

My code is

     private byte[] Invoke(Stream inputFileStream, CryptoAction action)
     {
        var msData = new MemoryStream();
        CryptoStream cs = null;

        try
        {
            long inputFileLength = inputFileStream.Length;
            var byteBuffer = new byte[4096];
            long bytesProcessed = 0;
            int bytesInCurrentBlock = 0;

            var csRijndael = new RijndaelManaged();
            switch (action)
            {
                case CryptoAction.Encrypt:
                    cs = new CryptoStream(msData, csRijndael.CreateEncryptor(this.Key, this.IV), CryptoStreamMode.Write);
                    break;

                case CryptoAction.Decrypt:
                    cs = new CryptoStream(msData, csRijndael.CreateDecryptor(this.Key, this.IV), CryptoStreamMode.Write);
                    break;
            }

            while (bytesProcessed < inputFileLength)
            {
                bytesInCurrentBlock = inputFileStream.Read(byteBuffer, 0, 4096);
                cs.Write(byteBuffer, 0, bytesInCurrentBlock);
                bytesProcessed += bytesInCurrentBlock;
            }
            cs.FlushFinalBlock();

            return msData.ToArray();
        }
        catch
        {
            return null;
        }
    }

In case of encrypting large files of size 60mb System.OutOfMemoryException is thrown and program crashes.My operating system is 64bit and have 8Gb of ram.

Answer 1

Try to get rid of all that buffer management code, that could be the cause of your problems… try to work with two streams (MemoryStream for volatile output is good):

using (FileStream streamInput = new FileStream(fileInput, FileMode.Open, FileAccess.Read))
{
    using (FileStream streamOutput = new FileStream(fileOutput, FileMode.OpenOrCreate, FileAccess.Write))
    {
        CryptoStream streamCrypto = null;
        RijndaelManaged rijndael = new RijndaelManaged();
        cspRijndael.BlockSize = 256;

        switch (CryptoAction)
        {
            case CryptoAction.ActionEncrypt:
                streamCrypto = new CryptoStream(streamOutput, rijndael.CreateEncryptor(this.Key, this.IV), CryptoStreamMode.Write);
                break;

            case CryptoAction.ActionDecrypt:
                streamCrypto = new CryptoStream(streamOutput, rijndael.CreateDecryptor(this.Key, this.IV), CryptoStreamMode.Write);
                break;
        }

        streamInput.CopyTo(streamCrypto);
        streamCrypto.Close();
    }
}