Question

How do request streams works with node.js (express or restify) ?

When a client who tries to upload an audio, mpeg or other binary file to the server, the request should be a Readable Stream on the server. that we could pipe into another stream using request.pipe() to for example get the file from the request, and then upload the file to amazon s3 using knox.

When I’m using an asynchronous authentication method part of the streamed data is being lost and the length doesn’t match with the transmitted content-length header anymore. Is there any way to avoid this behavior?
Is the data of request stream stored only in memory or does node.js store the data in some local temp folder?

var express = require('express'),
app = express(),
passport = require('passport'),
BasicStrategy = require('passport-http').BasicStrategy;

var users = [
    { id: 1, username: 'bob', password: 'secret', email: '[e-mail]' }
    , { id: 2, username: 'joe', password: 'birthday', email: '[e-mail]' }
];

function findByUsername(username, fn) {
    for (var i = 0, len = users.length; i < len; i++) {
        var user = users[i];
        if (user.username === username) {
            return fn(null, user);
        }
    }
    return fn(null, null);
}

passport.use(new BasicStrategy(
    function(username, password, done) {
        process.nextTick(function () {
            findByUsername(username, function(err, user) {
                if (err) { return done(err); }
                if (!user) { return done(null, false); }
                if (user.password != password) { return done(null, false); }
                return done(null, user);
            })
        });
    }));

app.configure(function() {
    app.use(express.logger());
    app.use(passport.initialize());
    app.use(app.router);
});    

app.post('/upload', 
    passport.authenticate('basic', { session: false }),
    function(req, res, next) {

        var dataLength = 0;

        req.on('data', function(chunk) {
            console.log('loading');
            dataLength += chunk.length;

        }).on('end', function() {
            console.log('load end');
            console.log('contentLength: %s', req.headers['content-length']);
            console.log('dataLength:  : %s', dataLength);
            res.send(200);

        }); 
    });

app.listen(8080, function() {
    console.log('server is running');
});

Answer 1

Lets see, where to begin:

• Yes, the request is a ReadStream that will emit data events as you said.
• The first chunk of data in the stream is stored in memory, there are no temporary files.

The general issue you are running into is that you need to capture the data emitted from the req and re-emit it once the authentication is completed. Connect provides helpers to do this for you. It looks like passport-http doesn’t currently use them.

If you install the pause module it can handle this for you.

Try something like this:

var pause = require('pause');

// Create our own middleware constructor.
var bufferedAuthenticate = function(){
    // Set up standard authenticate handler passing arguments through.
    var authMiddleware = passport.authenticate.apply(passport, arguments);

    // Pass our own middleware instead that wraps passport-http.
    return function(req, res, next){
        // Pause the request before authenticating.
        var obj = pause(req);
        authMiddleware(req, res, function(err){
          next(err);

          // Emit any cached data events that fired while authenticating.
          obj.resume();
        });
    };
};

app.post('/upload',
    bufferedAuthenticate('basic', { session: false }),
    function(req, res, next) {
        var dataLength = 0;

        req.on('data', function(chunk) {
            console.log('loading');
            dataLength += chunk.length;

        }).on('end', function() {
            console.log('load end');
            console.log('contentLength: %s', req.headers['content-length']);
            console.log('dataLength:  : %s', dataLength);
            res.send(200);
        }); 
    }
);