Question

I am trying to write text file after reading from a large tab delimited text file in a console application. Issue is that if I run many instances of this exe concurrently on a server, it gives me runtime error at TextWriter.WriteLine and then application crashes because of unhandled exception. This happens with all the instances. I fail to understand the reason for this behavior.. Is it because I am not using StringBuilder which will use memory dynamically ?

My code is as follows :

 StreamReader sr = new StreamReader(@FilePath, System.Text.Encoding.Default);
 string mainLine = sr.ReadLine();
 string[] fileHeaders = mainLine.Split(new string[] { "\t" }, StringSplitOptions.None);
 string newLine = "";

 System.IO.StreamWriter outFileSw = new System.IO.StreamWriter(@outFile);

 while (!sr.EndOfStream)
 {
    mainLine = sr.ReadLine();
    string[] originalLine = mainLine.Split(new string[] { "\t" }, StringSplitOptions.None);

   newLine = "";
   for (int i = 0; i < fileHeaders.Length; i++)
   {
      if(fileHeaders[i].Trim() != "")
       newLine = newLine + fileHeaders[i].Trim() + "=" + originalLine[i].Trim() + "&";
   }

outFileSw.WriteLine(newLine.Remove(newLine.Length - 1));
FileInfo fileInfo = new FileInfo(@outFile);

  if (fileInfo.Length > (1.3 * 1024.0 * 1024.0 * 1024.0)) // greater than 1.3 GB  
  {
          outFileSw.Close();
          outFileNumber = outFileNumber + 1;
          outFileSw = new System.IO.StreamWriter(@outFile + outFileNumber.ToString() + ".txt");
  }

}
outFileSw.Dispose();
sr.Close();
sr.Dispose();

Error details are :

Exception Message: The specified network name is no longer available.

Stack Trace:    at System.IO.__Error.WinIOError(Int32 errorCode, String maybeFullPath)
 at System.IO.FileStream.WriteCore(Byte[] buffer, Int32 offset, Int32 count)
 at System.IO.FileStream.Write(Byte[] array, Int32 offset, Int32 count)
 at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder)
 at System.IO.StreamWriter.Write(Char[] buffer, Int32 index, Int32 count)
 at System.IO.TextWriter.WriteLine(String value)
 at ExampleExe.ExampleProcess.FnFiles()


 Unhandled Exception: System.IO.IOException: The specified network name is no longer available.

 at System.IO.__Error.WinIOError(Int32 errorCode, String maybeFullPath)
 at System.IO.FileStream.WriteCore(Byte[] buffer, Int32 offset, Int32 count)
 at System.IO.FileStream.FlushWrite(Boolean calledFromFinalizer)
 at System.IO.FileStream.Dispose(Boolean disposing)
 at System.IO.FileStream.Finalize()

Thanks,
Kanu

Answer 1

Here is a FileWriter with StringBuilder buffer. It uses a separate thread to write while main thread keeps accumulating data. Writes 10MB buffer at a time. Buffered data is kept in Queue object; writer thread removes item from this Queue and writes it out full using with File.AppendAllText method.

private Queue<StringBuilder> writeQueue;
private bool isComplete;

public void FileWriter()
{
    this.isComplete = false;
    this.writeQueue = new Queue<StringBuilder>();

    var writer = new Action<string>(this.StartWriting);
    var writerAsync = writer.BeginInvoke(@"outputfile.txt", null, null);

    using (StreamReader sr = new StreamReader(@"inputfile.txt"))
    {
        var fileHeaders = sr.ReadLine()
            .Split('\t')
            .Where(i => !string.IsNullOrEmpty(i))
            .Select(j => j.Trim())
            .ToList();

        var buffer = new StringBuilder();
        while (!sr.EndOfStream)
        {
            var originalLine = sr.ReadLine()
                .Split('\t')
                .Where(i => !string.IsNullOrEmpty(i))
                .Select(j => j.Trim())
                .ToList();

            var line = new StringBuilder();
            //Must have same number of items
            if (originalLine.Count == fileHeaders.Count)
            {
                for (int i = 0; i < fileHeaders.Count(); i++)
                {
                    line.AppendFormat("{0}={1}&", fileHeaders[i], originalLine[i]);
                }
                line.AppendLine();
            }

            buffer.AppendLine(line.ToString());
            if (buffer.Length > 1024 * 1024 * 10)//approx 10MB 
            {
                lock (this.writeQueue)
                {
                    this.writeQueue.Enqueue(buffer);
                }
                buffer = new StringBuilder();
            }
        }
        //Queue any final remaining data
        if (buffer.Length>0) lock (this.writeQueue)
        {
            this.writeQueue.Enqueue(buffer);
        }
    }
    this.isComplete = true;
    writer.EndInvoke(writerAsync);
}

private void StartWriting(string outFilePath)
{
    while (!this.isComplete || this.writeQueue.Count > 0)
    {
        StringBuilder queuedItem;
        if (this.writeQueue.Count > 0)
        {
            lock (this.writeQueue)
            {
                queuedItem = this.writeQueue.Dequeue();
            }
            File.AppendAllText(outFilePath, queuedItem.ToString());
        }
        System.Threading.Thread.Sleep(5000); //Sleep 5sec
    }
}