Question

I have to import a large online csv (30k+ rows) file into my app’s database. How can I do this in another thread or an asynctask?

I am working on this code. It goes on onCreate:

final ProgressDialog Dialog = ProgressDialog.show(
            this, "Updating schedule", "This may take a few minutes...", true, false);

    Thread thread = new Thread(new Runnable() {
        public void run() {
            DatabaseHandler db = new DatabaseHandler(MainActivity.this);

            //**How do I begin transaction here?**

            URL myURL;
            try {
                myURL = new URL("http://www.meralco.com.ph/pms/pms.csv");

                BufferedReader so = new BufferedReader(new InputStreamReader(myURL.openStream()));
                while (true) {
                    String output = so.readLine();
                    if (output != null) {
                        String[] sched = output.split(",");
                        db.addRow(sched[INDEX_SIN], sched[INDEX_CITY], 
                                sched[INDEX_START_DATE], sched[INDEX_START_TIME], 
                                sched[INDEX_END_DATE], sched[INDEX_END_TIME], 
                                sched[INDEX_DETAILS], sched[INDEX_REASON]);
                    }
                    else {
                        break;
                    }
                }

                //**How do I close transaction here?**
                so.close();
            }
            catch (MalformedURLException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } 


            runOnUiThread(new Runnable() {
                public void run() {
                    while (!Dialog.isShowing());
                    Dialog.dismiss();
                }
            });
        }

    });
    thread.start();

DatabaseHandler.java:

public class DatabaseHandler{

public static final String KEY_SIN = "sched_sin";
public static final String KEY_CITY = "sched_city";
public static final String KEY_START_TIME = "sched_start_time";
public static final String KEY_START_DATE= "sched_start_date";
public static final String KEY_END_TIME = "sched_end_time";
public static final String KEY_END_DATE = "sched_end_date";
public static final String KEY_DETAILS = "sched_details";
public static final String KEY_REASON = "sched_reason";
public static final String KEY_ROWID = "_id";

private static final String TAG = "DatabaseHandler";
private SQLiteDatabase mDb;

private static final String DATABASE_NAME = "schedule_database";
private static final String DATABASE_TABLE = "schedule";
private static final int DATABASE_VERSION = 1;

private static final String DATABASE_CREATE =
    "create table " + DATABASE_TABLE + " (" + KEY_ROWID + " integer primary key autoincrement, "
    + KEY_SIN +" text not null, " + KEY_CITY + " text not null, " 
    + KEY_START_DATE +" text not null, " + KEY_START_TIME + " text not null, " 
    + KEY_END_DATE +" text not null, " + KEY_END_TIME + " text not null, "
    + KEY_DETAILS +" text not null, " + KEY_REASON + " text not null);";

private final Context mCtx;

public DatabaseHandler(Context ctx) {
    this.mCtx = ctx;

    // create or open the database
    DatabaseHelper helper = new DatabaseHelper(ctx);
    this.mDb = helper.getWritableDatabase();
}

public void addRow(String sin, String city, String start_date, String start_time, 
        String end_date,String end_time, String details, String reason)
{
    // this is a key value pair holder used by android's SQLite functions
    ContentValues values = new ContentValues();

    values.put(KEY_SIN, sin);
    values.put(KEY_CITY, city);
    values.put(KEY_START_DATE, start_date);
    values.put(KEY_START_TIME, start_time);
    values.put(KEY_END_DATE, end_date);
    values.put(KEY_END_TIME, end_time);
    values.put(KEY_DETAILS, details);
    values.put(KEY_REASON, reason);

    // ask the database object to insert the new data 
    try
    {
        this.mDb.insert(DATABASE_TABLE, null, values);
    }
    catch(Exception e)
    {
        Log.e("DB ERROR", e.toString()); // prints the error message to the log
        e.printStackTrace(); // prints the stack trace to the log
    }
}

public ArrayList<ArrayList<Object>> getAllRowsAsArrays()
{
    // create an ArrayList that will hold all of the data collected from
    // the database.
    ArrayList<ArrayList<Object>> dataArrays = new ArrayList<ArrayList<Object>>();

    // this is a database call that creates a "cursor" object.
    // the cursor object store the information collected from the
    // database and is used to iterate through the data.
    Cursor cursor;

    try
    {
        // ask the database object to create the cursor.
        cursor = this.mDb.query(
                DATABASE_TABLE,
                new String[]{KEY_ROWID, KEY_SIN, KEY_CITY, KEY_START_DATE, KEY_START_TIME, KEY_END_DATE, KEY_END_TIME, KEY_DETAILS, KEY_REASON},
                null, null, null, null, null
        );

        // move the cursor's pointer to position zero.
        cursor.moveToFirst();

        // if there is data after the current cursor position, add it
        // to the ArrayList.
        if (!cursor.isAfterLast())
        {
            do
            {
                ArrayList<Object> dataList = new ArrayList<Object>();

                dataList.add(cursor.getLong(0));
                dataList.add(cursor.getString(1));
                dataList.add(cursor.getString(2));
                dataList.add(cursor.getString(3));
                dataList.add(cursor.getString(4));
                dataList.add(cursor.getString(5));
                dataList.add(cursor.getString(6));
                dataList.add(cursor.getString(7));
                dataList.add(cursor.getString(8));

                dataArrays.add(dataList);
            }
            // move the cursor's pointer up one position.
            while (cursor.moveToNext());
        }
    }
    catch (SQLException e)
    {
        Log.e("DB Error", e.toString());
        e.printStackTrace();
    }

    // return the ArrayList that holds the data collected from
    // the database.
    return dataArrays;
}

private static class DatabaseHelper extends SQLiteOpenHelper {  

    DatabaseHelper(Context context) {
        super(context, DATABASE_NAME, null, DATABASE_VERSION);
    }

    @Override
    public void onCreate(SQLiteDatabase db) {
        Log.i(TAG, "Creating DataBase: " + DATABASE_CREATE);
        db.execSQL(DATABASE_CREATE);
    }

    @Override
    public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
        Log.w(TAG, "Upgrading database from version " + oldVersion + " to "
                + newVersion + ", which will destroy all old data");
        db.execSQL("DROP TABLE IF EXISTS " + DATABASE_TABLE);
        onCreate(db);
    }
}
}

EDIT
Also, I am having trouble using beginTransaction() and endTransaction because I can’t use it inside the thread. It says the methods below are undefined….

public void beginTransaction(){
    this.mDb.beginTransaction();
}

public void endTransaction(String sql){
    this.mDb.endTransaction();
}   

Answer 1

Below is the code that worked for me. I noticed downloading is faster when using Threads instead of AsyncTask so I chose the former. And as @JoxTraex pointed out, I used a csv parser (in this case, I used the reader of OpenCSV) instead of a full working block. It eliminated my problems of splitting at every comma even if a value is enclosed in a pair of double quotes.

Thread downloadThread = new Thread(new Runnable() {

    public void run() {

        db = new DatabaseHandler(MainActivity.this);
        db.beginTransaction();

        try {
            URL url = new URL(csvFileName);

            Log.i("dl", "start");

            InputStream input = url.openStream();
            CSVReader reader = new CSVReader(new InputStreamReader(input));

            String [] sched;
            while ((sched = reader.readNext()) != null) {
                if(sched[INDEX_CITY].equals("")) sched[INDEX_CITY]="OTHERS";
                try {

                    db.addRow(sched[INDEX_SIN], sched[INDEX_CITY], 
                            sched[INDEX_START_DATE], sched[INDEX_START_TIME], 
                            sched[INDEX_END_DATE], sched[INDEX_END_TIME], 
                            sched[INDEX_DETAILS], sched[INDEX_REASON]);
                } catch (IndexOutOfBoundsException e) {
                    db.addRow(sched[INDEX_SIN], sched[INDEX_CITY], 
                            sched[INDEX_START_DATE], sched[INDEX_START_TIME], 
                            sched[INDEX_END_DATE], sched[INDEX_END_TIME], 
                            "", sched[INDEX_REASON]);
                    //e.printStackTrace();
                }
            }
            input.close();
            Log.i("dl", "finished");

        } catch (MalformedURLException e) {
            e.printStackTrace();
            db.endTransaction();
        } catch (IOException e) {
            e.printStackTrace();
            db.endTransaction();
        } 
        Log.d("Count", ""+db.count());
        db.setTransactionSuccessful();
        db.endTransaction();

        writeUploadDateInTextFile();

    }

});

In my database handler class,

public void beginTransaction(){
    this.mDb.beginTransaction();
}

public void endTransaction(String sql){
    this.mDb.endTransaction();
}