Question

I made function in c# to read line by line and then load lines to sqlite (s3db).

private void LoadFromDictionary()
    {
        Encoding enc = Encoding.GetEncoding(1250);
        using (StreamReader r = new StreamReader("c:\\Temp2\\dictionary.txt", enc))
        {
            string line = "";
            while ((line = r.ReadLine()) != null)
            {
                line = line.Trim();
                AddWord(line);
            }
        }
        MessageBox.Show("Finally :P", "Info");
    }

    private void AddWord(string w)
    {
        String insSQL = "insert into Words values(\"" + w + "\")";
        String strConn = @"Data Source=C:\Temp2\dictionary.s3db";
        SQLiteConnection conn = new SQLiteConnection(strConn);
        SQLiteDataAdapter da = new SQLiteDataAdapter(insSQL, strConn);
        da.Fill(dt);
        dataGridView1.DataSource = dt.DefaultView;
    }

But is it any faster way? I created table by sqlite administrator application.
Can sqlite load itself file and make it as a table?

---

I am talking about 3+ millions words (one word in one line).

PS. please correct my topic if there is something wrong 🙂

Answer 1

Yes, there is a much, much faster method using the following techniques:

1) Only open a connection to the database one time

2) Use a parameterized command for better performance and lower overhead (don’t have to use new strings on each pass).

3) Wrap the entire operation in a transaction. As a general rule, this will improve your performance.

Note that I do not show transaction rollback or closing the connection, which are also best practices that should be implemented.

private void LoadFromDictionary()
    {
            Encoding enc = Encoding.GetEncoding(1250);
            string strConn = @"Data Source=C:\Temp2\dictionary.s3db";
            SqliteConnection conn = new SqliteConnection(strConn);

            conn.Open();

            string insSQL = "insert or ignore into wyrazy values(@Word)";

            DbCommand oCommand = conn.CreateCommand();

            oCommand.Connection = conn;
            oCommand.CommandText = insSQL;

            DbParameter oParameter = oCommand.CreateParameter();
            oParameter.Name = "@Word";
            oParameter.DbType = DbType.String;
            oParameter.Size = 100;

            oCommand.Parameters.Add(oParameter);

            DbTransaction oTransaction = conn.BeginTransaction();

            using (StreamReader r = new StreamReader("c:\\Temp2\\dictionary.txt", enc))
            {
                string line = "";
                while ((line = r.ReadLine()) != null)
                {
                    line = line.Trim();
                    if (!string.IsNullOrEmpty(line)) {
                      oParameter.Value = line;
                      oCommand.ExecuteNonQuery();
                    }
                }
            }
            oTransaction.Commit();

            conn.Close();

            MessageBox.Show("Finally :P", "Info");
        }