Question

I have a C# console app that I’m trying to create that processes all the files in a given directory and writes output to another given directory. I want to process the input files X bytes at a time.

namespace FileConverter
{
    class Program
    {
        static void Main(string[] args)
        {
            string srcFolder = args[0];  
            string destFolder = args[1];   
            string[] srcFiles = Directory.GetFiles(srcFolder);
            for (int s = 0; s < srcFiles.Length; s++)
            {
                byte[] fileBuffer;
                int numBytesRead = 0;
                int readBuffer = 10000;
                FileStream srcStream = new FileStream(srcFiles[s], FileMode.Open, FileAccess.Read);
                int fileLength = (int)srcStream.Length;

                string destFile = destFolder + "\\" + Path.GetFileName(srcFiles[s]) + "-processed";
                FileStream destStream = new FileStream(destFile, FileMode.OpenOrCreate, FileAccess.Write);

                //Read and process the source file by some chunk of bytes at a time
                while (numBytesRead < fileLength)
                {
                    fileBuffer = new byte[readBuffer];

                    //Read some bytes into the fileBuffer
                    //TODO: This doesn't work on subsequent blocks
                    int n = srcStream.Read(fileBuffer, numBytesRead, readBuffer);

                    //If we didn't read anything, there's no more to process
                    if (n == 0)
                        break;

                    //Process the fileBuffer
                    for (int i = 0; i < fileBuffer.Length; i++)
                    {
                        //Process each byte in the array here
                    }
                    //Write data
                    destStream.Write(fileBuffer, numBytesRead, readBuffer);
                    numBytesRead += readBuffer;
                }
                srcStream.Close();
                destStream.Close();
            }
        }
    }
}

I’m running into an error at execution time at:

//Read some bytes into the fileBuffer
//TODO: This doesn't work on subsequent blocks
int n = srcStream.Read(fileBuffer, numBytesRead, readBuffer);

I don’t want to load the entire file into memory, as it could possibly be many gigabytes in size. I really want to be able to read some number of bytes, process them, write them out to a file, and then read in the next X bytes and repeat.

It gets through one iteration of the loop, and then dies on the second. The error I get is:

“Offset and length were out of bounds for the array or count is greater than the number of elements from index to the end of the source collection.”

The sample file I’m working with is about 32k.

Can anyone tell me what I’m doing wrong here?

Answer 1

The second parameter to Read is not the offset into the file – it is the offset into the buffer at which to start writing data. So just pass 0.

Also, don’t assume the buffer is filled each time: you should only process “n” bytes from the buffer. And the buffer should be reused between iterations.

If you need to read exactly a number of bytes:

static void ReadOrThrow(Stream source, byte[] buffer, int count) {
     int read, offset = 0;
     while(count > 0 && (read = source.Read(buffer, offset, count)) > 0) {
        offset += read;
        count -= read;
    }
    if(count != 0) throw new EndOfStreamException();
}

Note that Write works similarly, so you need to pass 0 as the offset and n as the count.