Question

I have a program that reads approximately 2million rows from a database into a List.
Each row is a location that contains information such as geographic co-ordinates.

Once data is added to the List I use a foreach loop and grab the co-ordinates to create a kml file. The loop encounters an OutOfMemoryException error when the number of rows is large (but works perfectly otherwise).

Any suggestions on how to handle this so that the program can work with very large sets of data? The kml library is SharpKML.

I am still new to C# so please go easy!

This is the loop:

            using (SqlConnection conn = new SqlConnection(connstring))
        {
            conn.Open();
            SqlCommand cmd = new SqlCommand(select, conn);

            using (cmd)
            {
                SqlDataReader reader = cmd.ExecuteReader();
                while (reader.Read())
                {
                    double lat = reader.GetDouble(1);
                    double lon = reader.GetDouble(2);
                    string country = reader.GetString(3);
                    string county = reader.GetString(4);
                    double TIV = reader.GetDouble(5);
                    double cnpshare = reader.GetDouble(6);
                    double locshare = reader.GetDouble(7);

                    //Add results to list
                    results.Add(new data(lat, lon, country, county, TIV, cnpshare, locshare));
                }
                reader.Close();
            }
            conn.Close();
        }

            int count = results.Count();
            Console.WriteLine("number of rows in results = " + count.ToString());

            //This code segment generates the kml point plot

            Document doc = new Document();
            try
            {
                foreach (data l in results)
                {
                    Point point = new Point();
                    point.Coordinate = new Vector(l.lat, l.lon);

                    Placemark placemark = new Placemark();
                    placemark.Geometry = point;
                    placemark.Name = Convert.ToString(l.tiv);

                    doc.AddFeature(placemark);

                }
            }
            catch(OutOfMemoryException e)
            {
                throw e;
            }

This is the class uused in the List

        public class data
    {
        public double lat { get; set; }
        public double lon { get; set; }
        public string country { get; set; }
        public string county { get; set; }
        public double tiv { get; set; }
        public double cnpshare { get; set; }
        public double locshare { get; set; }

        public data(double lat, double lon, string country, string county, double tiv, double cnpshare,
            double locshare)
        {
            this.lat = lat;
            this.lon = lon;
            this.country = country;
            this.county = county;
            this.tiv = tiv;
            this.cnpshare = cnpshare;
            this.locshare = locshare;
        }

    }

Answer 1

If there is no big delay in populating list with data from database and you did not mentioned problems with populating list with data, why not immediately create your Point and Placemark object. Code is below.

    var doc = new Document();

    using (SqlConnection conn = new SqlConnection(connstring))
    {
        conn.Open();
        SqlCommand cmd = new SqlCommand(select, conn);

        using (cmd)
        {
            var reader = cmd.ExecuteReader();
            while (reader.Read())
            {
                double lat = reader.GetDouble(1);
                double lon = reader.GetDouble(2);
                string country = reader.GetString(3);
                string county = reader.GetString(4);
                double TIV = reader.GetDouble(5);
                double cnpshare = reader.GetDouble(6);
                double locshare = reader.GetDouble(7);

                var point = new Point();
                point.Coordinate = new Vector(lat , lon );

                var placemark = new Placemark();
                placemark.Geometry = point;
                placemark.Name = Convert.ToString(TIV);

                doc.AddFeature(placemark);

            reader.Close();
        }
        conn.Close();
    }

If there is no good reason for retrieving so many data in memory, try with some lazy loading approach.