Parse and Split Very Large Text Files Without Out of Memory Exceptions

By Peter Bromberg

From time to time you may have the need to read a very large file and split into an array. Streamreader is your friend here.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace SplitLargeMailboxFile
{
    class Program
    {
         static void Main(string[] args)
        {
           List<string> emails = new List<string>();
             try
            {
                // the file is in the /bin/debug folder for simplicity
                 using (StreamReader sr = new StreamReader(System.Environment.CurrentDirectory+@"\Main.mbx"))
                 {
                     String line;
                     string tmpText = String.Empty;
                     while ((line = sr.ReadLine()) != null)
                     {
                          if (!line.Contains("[message truncated]"))
                        {
                            tmpText += line;
                         }
                         else
                         {
                               emails.Add(tmpText);
                            tmpText = String.Empty;
                           Console.WriteLine(".");
                         }
                         
                      }
                 }
             }
             catch (Exception e)
            {
               Console.WriteLine("The file could not be read:");
               Console.WriteLine(e.Message);
            }
        }
    }
}

Parse and Split Very Large Text Files Without Out of Memory Exceptions  (1979 Views)