Question

The following code tries to load from a csv file to a mongodb instance running on localhost.

Problem – It loads a different number of documents on every run (always less than total number of records in csv).

var csv = require('csv');

var server = new Server('localhost', 27017, {auto_reconnect: true, poolSize: 1});
var db = new Db('test', server);

db.open(function(err, db, onemore) {
  if(!err) {
    //Database connection is established.
    db.collection('teststocks', function(err, collection) {
      if(!err) {
        // Stocks collection is connected, open the file and insert the doc
        console.log("Trying to load from " + process.argv[2]);
        csv()
          .fromPath(process.argv[2], {
            columns: true
          })
          .on('data', function(data, index) {
            //data.stock = process.argv[2].substring(process.argv[2].lastIndexOf('/') + 1, process.argv[2].lastIndexOf('.'));
            collection.insert(data, {safe: true}, function(error, collection){
                    if ( error ) { console.log("Error inserting record : " + error); }
            });
            console.log("Inserted data for " + index);
          })
          .on('error', function(error) {
            db.close();
            console.log("Error: " + error);
          })
          .on('end', function(count) {
            console.log("Finished all writing.");
            db.close();
          });
      }
  });
  }
});

P.S: I am able to load the data using the mongoimport utility, but being a newcomer to node.js and mongodb; I’d like to understand the mistake I am making in the code above.

Answer 1

It’s likely because you’re closing the database connection after the csv() finishes reading. But, as collection.insert is asynchronous, there’s no guarantee each call has finished before the csv is done and db.close() is called.

One option is to collect the csv data into an Array, then insert all of them at once:

var docs = [];

csv()
  // ...
  .on('data', function (data, index) {
    docs.push(data); // or possibly: docs[index] = data;
  })
  // ...
  .on('end', function () {
    console.log("Finished reading CSV.");    

    collection.insert(docs, { safe: true }, function (error, inserted) {
      console.log("Finished all writing.");
      db.close();
    });
  });