Question

I have an script that adds products to a remote database using a RESTful API in Node.js. It works well, but I would like to change the priority HTTP requests are processed. Here is some code to better illustrate what I am trying to do:

step(
    function initializeCategories() {
        createCategories(products, this);
    },
    function createProducts(err, categoriesHash) {
        console.log("\nCreating products:");
        console.log("==================");

        var group = this.group(),
            productDoneCallback;

        products.forEach(function (product) {
            product.categories = categoriesHash[product.category + "/" + product.make + "/" + product.model];
            productDoneCallback = group();

            step(
                function createProduct() {
                    postProduct(convertToBigCommerceObj(product), this);
                },
                function overwriteProduct(err, product, allowOverwrite) {
                    if (err) {
                        console.log(err);
                    }

                    allowOverwrite = allowOverwrite || false;

                    if (allowOverwrite) {
                        updateProduct(product, this);
                    } else {
                        this(err, product);
                    }
                },
                function addExtraInfo(err, product) {
                    addImage(product, productDoneCallback);
                }
            );
        });
    },
    function printStats(err) {
        if (err) {
            logError(err);
        }

        var endTime = +new Date(),
            duration = endTime - startTime;

        console.log("\nFinished after " + (duration / 1000 / 60) + " minutes");
        console.log(productsAdded + " Products added successfully");
        console.log(productsUpdated + " Products updated successfully");
        console.log(productsSkipped + " Products skipped");
        console.log("Average time (milliseconds) per product was : " + (duration / totalNumProducts ));
        console.log("For more information see error log (error.log)" );
    }
);

In this code the product images are always added last after all products have been added. This is because the forEach loop puts all of the postProduct requests on the node event queue right away. After the first product is done has been posted to the server another entry is added to the end of the queue to add that product’s image. Instead, I would like that new entry to float to the top of the queue and be the next entry to be processed (not another product post, that can wait).

I realize that to do that I need a priority queue. I am just not sure how I can achieve this in Node and Javascript.

Update: after finding an implementation of a PriorityQueue at https://github.com/STRd6/PriorityQueue.js, the real trouble is processing the queue asynchronously every time a request completes. Or more like everytime there is an available http channel that has been freed we need to take the highest priority item.

Answer 1

Whenever you heard “priority queue”, you should think about having a heap data structure. They are not the only way to do that, but they are simple to implement and so are a good first goto.

See http://eloquentjavascript.net/appendix2.html for a random implementation. They assume that when you create the heap, you pass in a function that takes an element and returns its priority. In your case you could store elements like [priority, object] and initialize your heap wth a function like function (x) {return x[0]}.