Question

I am running a background thread in my asp.net web service application. This thread’s responsibility is to hit database after a specific time and update a datatable in the Cache. The data table has around 500K rows. In task manager when I look in processes, the web dev server for first time consumes around 300,000K on next time it goes to 500,000K and some times it reaches above 1,000,000K and sometimes drop back to 500,000-600,000K. As I am doing work on my local machine so data in database is not changing.
Can anyone please guide me what I am doing wrong in the code:

protected void Application_Start(object sender, EventArgs e)
    {
        Thread obj = new Thread(new ThreadStart(AddDataInCache));
        obj.IsBackground = true;
        obj.Start();
    }

private void AddDataInCache()
    {
        Int32 iCount = 0;
        while (true)
        {
            MyCollection _myCollection = new MyCollection();
            DataTable dtReferences = null;
            DataTable dtMainData = null;
            try
            {
                dtMainData = _myCollection.GetAllDataForCaching(ref dtReferences);

                HttpRuntime.Cache.Insert("DATA_ALL_CACHING", dtMainData, null,
                    Cache.NoAbsoluteExpiration, Cache.NoSlidingExpiration,
                    CacheItemPriority.Default, null);

                HttpRuntime.Cache.Insert("DATA_REFERENCES_CACHING", dtReferences, null,
                    Cache.NoAbsoluteExpiration, Cache.NoSlidingExpiration,
                    CacheItemPriority.NotRemovable, null
                    );
            }
            catch (Exception ex)
            {

            }
            finally
            {
                if (_myCollection != null)
                    _myCollection = null;

            }
            iCount++;
            Thread.Sleep(18000);
        }
    }

In GetAllDataForCaching I am getting a SqlDataReader from my Data Access layer as:

public DataTable GetAllDataForCaching(ref DataTable dReferenceTable)
{
      DataTable dtReturn = new DataTable();
      SqlDataReader dReader = null;
      try
      {
            dReader = SqlHelper.ExecuteReader(CommandType.StoredProcedure, "[GetDataForCaching]", null);
            if (dReader != null && dReader.HasRows)
            {
                  dtReturn.Load(dReader);
                  dReferenceTable = new DataTable();
                  if (dReader.HasRows)
                {
                    DataTable dtSchema = dReader.GetSchemaTable();
                    List<DataColumn> listCols = new List<DataColumn>();

                    if (dtSchema != null)
                    {
                        foreach (DataRow drow in dtSchema.Rows)
                        {
                            string columnName = System.Convert.ToString(drow["ColumnName"]);
                            DataColumn column = new DataColumn(columnName, (Type)(drow["DataType"]));
                            column.Unique = (bool)drow["IsUnique"];
                            column.AllowDBNull = (bool)drow["AllowDBNull"];
                            column.AutoIncrement = (bool)drow["IsAutoIncrement"];
                            listCols.Add(column);
                            dReferenceTable.Columns.Add(column);
                        }
                    }

                    while (dReader.Read())
                    {
                        DataRow dataRow = dReferenceTable.NewRow();
                        for (int i = 0; i < listCols.Count; i++)
                        {
                            dataRow[((DataColumn)listCols[i])] = dReader[i];
                        }
                        dReferenceTable.Rows.Add(dataRow);
                    }
                }
            }
      }
      finally
        {
            if (dReader != null)
            {
                if (dReader.IsClosed == false)
                    dReader.Close();
                dReader = null;
            }
        }
    return dtReturn;
}

I am using Visual Studio 2008.

Answer 1

I have done it by putting following code before Thread.Sleep(18000);

GC.Collect();
GC.WaitForPendingFinalizers();

It is keeping the memory in control so far.