Question

I am new to javascript and node.js and this is my first post, so please bear with me.

I am using ntwitter to get all previous tweets of a specific user.

My problem is that if the user has more than 200 tweets, I need to create a loop and I am not sure if I do it right.

This is the async function that gets the 200 latest tweets:

exports.getUserTimeline = function(user, callback) {

  twit.getUserTimeline({ screen_name: user, count: 200 }, function(err, data) {
    if (err) { 
      return callback(err);
    }
    callback(err, data);  
  });
}

I found a solution to do this using a recursive function, but it’s quite ugly.. How can I improve it ?

exports.getUserHistory = function(user, callback) {
  recursiveSearch(user, callback);
  function recursiveSearch(user, callback, lastId, data) {
    var data = data || []
      , args = {screen_name: user, count: 200};

    if(typeof lastId != "undefined") args.max_id = lastId;

    twit.getUserTimeline(args, function(err, subdata) {
      if (err) { 
        console.log('Twitter search failed!');
        return callback(err);
      }
      if (data.length !== 0) subdata.shift();
      data = data.concat(subdata);
      var lastId = parseInt(data[data.length-1].id_str);
      if (subdata.length !== 0) {
        recursiveSearch(user, callback, lastId, data);
      } else {
        callback(err, data);
      }
    });
  }
}

Thank’s a lot!

Update: This is the improved (refactored) function suggested by hunterloftis with two modifications:

1. property max_id should not be specified on the first iteration
2. the case where the user exists but no tweets have been posted must be handled

code:

function getUserHistory(user, done) {
  var data = [];
  search();

  function search(lastId) {
    var args = {
      screen_name: user,
      count: 200,
      include_rts: 1
    };
    if(lastId) args.max_id = lastId;

    twit.getUserTimeline(args, onTimeline);

    function onTimeline(err, chunk) {
      if (err) {
        console.log('Twitter search failed!');
        return done(err);
      }

      if (!chunk.length) {
        console.log('User has not tweeted yet');
        return done(err);
      }

      //Get rid of the first element of each iteration (not the first time)
      if (data.length) chunk.shift();

      data = data.concat(chunk);
      var thisId = parseInt(data[data.length - 1].id_str);

      if (chunk.length) return search(thisId);
      console.log(data.length + ' tweets imported');
      return done(undefined, data);
    }
  }
}

When retrieving tweets I noticed that my tweet count wasn’t always the same as the ‘statuses_count’ property of the user. It took me some time to figure out that this difference corresponds to the number of deleted tweets 🙂

Answer 1

Does your recursive function work? Doesn’t look too bad to me. I might refactor it just a little into something more like this:

function getUserHistory(user, done) {
  var data = [];
  search();

  function search(lastId) {
    var args = {
      screen_name: user,
      count: 200,
      max_id: lastId
    };

    twit.getUserTimeline(args, onTimeline);

    function onTimeline(err, chunk) {
      if (err) {
        console.log('Twitter search failed!');
        return done(err);
      }

      if (data.length) chunk.shift(); // What is this for?
      data = data.concat(chunk);
      var thisId = parseInt(data[data.length - 1].id_str);

      if (chunk.length) return search(thisId);
      return done(undefined, data);
    }
  }
}