Question

I am just learning server-side JavaScript so please bear with any glaring mistakes I’ve made.

I am trying to write a file parser that operates on HTML files in a directory and returns a JSON string once all files have been parsed. I started it with a single file and it works fine. it loads the resource from Apache running on the same machine, injects jquery, does the parsing and returns my JSON.

var request = require('request'),
    jsdom = require('jsdom'),
    sys = require('sys'),
    http = require('http');

http.createServer(function (req, res) {
    request({uri:'http://localhost/tfrohe/Car3E.html'}, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            var window = jsdom.jsdom(body).createWindow();
            jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
              // jQuery is now loaded on the jsdom window created from 'body'
                var emps = {};
                jquery("tr td img").parent().parent().each(function(){
                    var step = 0;
                    jquery(this).children().each(function(index){
                        if (jquery(this).children('img').attr('src') !== undefined) {
                            step++;
                            var name = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                            var name_parts = name.split(",");
                            var last = name_parts[0];
                            var name_parts = name_parts[1].split(/\u00a0/g);
                            var first = name_parts[2];
                            emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                        }
                    });
                });
                emps = JSON.stringify(emps);
                //console.log(emps);
                res.writeHead(200, {'Content-Type': 'text/plain'});
                res.end(emps);


            });
        } else {
            res.writeHead(200, {"Content-Type": "text/plain"});
            res.end("empty");
            //console.log(response.statusCode);
        }
    });
}).listen(8124);

Now I am trying to extend this to using the regular file system (fs) and get all HTML files in the directory and parse them the same way and return a single combined JSON object once all files have been parsed. Here is what I have so far but it does not work.

var sys = require("sys"),
    fs = require("fs"),
    jsdom = require("jsdom"),
    emps = {};
    //path = '/home/inet/www/media/employees/';

readDirectory = function(path) {
    fs.readdir(path, function(err, files) {
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });
        var count = htmlfiles.length;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step++;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                });
            });
        });
    });
}

readDirectory('/home/inet/www/media/employees/', function() {
    console.log(emps);
});

In this particular case, there are 2 html files in the directory. If i console.log(emps) during the htmlfiles.forEach() it shows me the results from the first file then the results for both files together the way I expect. how do I get emps to be returned to readDirectory so i can output it as desired?

Completed Script

After the answers below, here is the completed script with a httpServer to serve up the detail.

var sys = require('sys'),
    fs = require("fs"),
    http = require('http'),
    jsdom = require('jsdom'),
    emps = {};



    var timed = setInterval(function() {
        emps = {};
        readDirectory('/home/inet/www/media/employees/', function(emps) {
        });
    }, 3600000);

readDirectory = function(path, callback) {
    fs.readdir(path, function(err, files) {
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });
        var count = htmlfiles.length;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    var imagecount = jquery("tr td img").length;
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step += 1;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                    count -= 1;
                    if (count <= 0) {
                        callback(JSON.stringify(emps));
                    }
                });
            });

        });
    });
}

var init = readDirectory('/home/inet/www/media/employees/', function(emps) {

        });


http.createServer(function (req, res) {
    res.writeHead(200, {'Content-Type': 'text/plain'});
    res.end(JSON.stringify(emps));
}).listen(8124);

Answer 1

That sure is a lot of code a couple of mistakes.

1. You’re never calling the callback function you supply to readDirectory
2. You need to keep track of the files you have parsed, when you parsed all of them, call the callback and supply the emps

This should work:

var sys = require("sys"),
    fs = require("fs"),
    jsdom = require("jsdom"),
    //path = '/home/inet/www/media/employees/';

// This is a nicer way
function readDirectory(path, callback) {
    fs.readdir(path, function(err, files) {

        // make this local
        var emps = {};
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });

        // Keep track of the number of files we have parsed
        var count = htmlfiles.length;
        var done = 0;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step++;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                    // As soon as all have finished call the callback and supply emps
                    done++;
                    if (done === count) {
                        callback(emps);
                    }   
                });
            });
        });
    });
}

readDirectory('/home/inet/www/media/employees/', function(emps) {
    console.log(emps);
});