How to avoid long nesting of asynchronous functions in Node.js
JavascriptAsynchronousFunctional Programmingnode.jsJavascript Problem Overview
I want to make a page that displays some data from a DB, so I have created some functions that get that data from my DB. I'm just a newbie in Node.js, so as far as I understand, if I want to use all of them in a single page (HTTP response) I'd have to nest them all:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
getSomeDate(client, function(someData) {
html += "<p>"+ someData +"</p>";
getSomeOtherDate(client, function(someOtherData) {
html += "<p>"+ someOtherData +"</p>";
getMoreData(client, function(moreData) {
html += "<p>"+ moreData +"</p>";
res.write(html);
res.end();
});
});
});
If there are many functions like that, then the nesting becomes a problem.
Is there a way to avoid this? I guess it has to do with how you combine multiple asynchronous functions, which seems to be something fundamental.
Javascript Solutions
Solution 1 - Javascript
Interesting observation. Note that in JavaScript you can normally replace inline anonymous callback functions with named function variables.
The following:
http.createServer(function (req, res) {
// inline callback function ...
getSomeData(client, function (someData) {
// another inline callback function ...
getMoreData(client, function(moreData) {
// one more inline callback function ...
});
});
// etc ...
});
Could be rewritten to look something like this:
var moreDataParser = function (moreData) {
// date parsing logic
};
var someDataParser = function (someData) {
// some data parsing logic
getMoreData(client, moreDataParser);
};
var createServerCallback = function (req, res) {
// create server logic
getSomeData(client, someDataParser);
// etc ...
};
http.createServer(createServerCallback);
However unless you plan to reuse to callback logic in other places, it is often much easier to read inline anonymous functions, as in your example. It will also spare you from having to find a name for all the callbacks.
In addition note that as @pst noted in a comment below, if you are accessing closure variables within the inner functions, the above would not be a straightforward translation. In such cases, using inline anonymous functions is even more preferable.
Solution 2 - Javascript
Kay, simply use one of these modules.
It will turn this:
dbGet('userIdOf:bobvance', function(userId) {
dbSet('user:' + userId + ':email', '[email protected]', function() {
dbSet('user:' + userId + ':firstName', 'Bob', function() {
dbSet('user:' + userId + ':lastName', 'Vance', function() {
okWeAreDone();
});
});
});
});
Into this:
flow.exec(
function() {
dbGet('userIdOf:bobvance', this);
},function(userId) {
dbSet('user:' + userId + ':email', '[email protected]', this.MULTI());
dbSet('user:' + userId + ':firstName', 'Bob', this.MULTI());
dbSet('user:' + userId + ':lastName', 'Vance', this.MULTI());
},function() {
okWeAreDone()
}
);
Solution 3 - Javascript
For the most part, I'd agree with Daniel Vassallo. If you can break up a complicated and deeply nested function into separate named functions, then that is usually a good idea. For the times when it makes sense to do it inside a single function, you can use one of the many node.js async libraries available. People have come up with lots of different ways to tackle this, so take a look at the node.js modules page and see what you think.
I've written a module for this myself, called async.js. Using this, the above example could be updated to:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
async.series({
someData: async.apply(getSomeDate, client),
someOtherData: async.apply(getSomeOtherDate, client),
moreData: async.apply(getMoreData, client)
},
function (err, results) {
var html = "<h1>Demo page</h1>";
html += "<p>" + results.someData + "</p>";
html += "<p>" + results.someOtherData + "</p>";
html += "<p>" + results.moreData + "</p>";
res.write(html);
res.end();
});
});
One nice thing about this approach is that you can quickly change your code to fetch the data in parallel by changing the 'series' function to 'parallel'. What's more, async.js will also work inside the browser, so you can use the same methods as you would in node.js should you encounter any tricky async code.
Hope that's useful!
Solution 4 - Javascript
You could use this trick with an array rather than nested functions or a module.
Far easier on the eyes.
var fs = require("fs");
var chain = [
function() {
console.log("step1");
fs.stat("f1.js",chain.shift());
},
function(err, stats) {
console.log("step2");
fs.stat("f2.js",chain.shift());
},
function(err, stats) {
console.log("step3");
fs.stat("f2.js",chain.shift());
},
function(err, stats) {
console.log("step4");
fs.stat("f2.js",chain.shift());
},
function(err, stats) {
console.log("step5");
fs.stat("f2.js",chain.shift());
},
function(err, stats) {
console.log("done");
},
];
chain.shift()();
You can extend the idiom for parallel processes or even parallel chains of processes:
var fs = require("fs");
var fork1 = 2, fork2 = 2, chain = [
function() {
console.log("step1");
fs.stat("f1.js",chain.shift());
},
function(err, stats) {
console.log("step2");
var next = chain.shift();
fs.stat("f2a.js",next);
fs.stat("f2b.js",next);
},
function(err, stats) {
if ( --fork1 )
return;
console.log("step3");
var next = chain.shift();
var chain1 = [
function() {
console.log("step4aa");
fs.stat("f1.js",chain1.shift());
},
function(err, stats) {
console.log("step4ab");
fs.stat("f1ab.js",next);
},
];
chain1.shift()();
var chain2 = [
function() {
console.log("step4ba");
fs.stat("f1.js",chain2.shift());
},
function(err, stats) {
console.log("step4bb");
fs.stat("f1ab.js",next);
},
];
chain2.shift()();
},
function(err, stats) {
if ( --fork2 )
return;
console.log("done");
},
];
chain.shift()();
Solution 5 - Javascript
I like async.js a lot for this purpose.
The issue is solved by waterfall command:
> waterfall(tasks, [callback])
> Runs an array of functions in series, each passing their results to the next in the array. However, if any of the functions pass an error to the callback, the next function is not executed and the main callback is immediately called with the error.
> Arguments
> tasks - An array of functions to run, each function is passed a callback(err, result1, result2, ...) it must call on completion. The first argument is an error (which can be null) and any further arguments will be passed as arguments in order to the next task. callback(err, [results]) - An optional callback to run once all the functions have completed. This will be passed the results of the last task's callback.
Example
async.waterfall([
function(callback){
callback(null, 'one', 'two');
},
function(arg1, arg2, callback){
callback(null, 'three');
},
function(arg1, callback){
// arg1 now equals 'three'
callback(null, 'done');
}
], function (err, result) {
// result now equals 'done'
});
As for the req,res variables, they will be shared within the same scope as function(req,res){} which enclosed the whole async.waterfall call.
Not only so, async is very clean. What I means is that I change a lot of cases like this:
function(o,cb){
function2(o,function(err, resp){
cb(err,resp);
})
}
To first:
function(o,cb){
function2(o,cb);
}
Then to this:
function2(o,cb);
Then to this:
async.waterfall([function2,function3,function4],optionalcb)
It also allows many premade functions prepared for async to be called from util.js very fast. Just chain up what you want to do, make sure o,cb is universally handled. This speeds up the whole coding process a lot.
Solution 6 - Javascript
What you need is a bit of syntactic sugar. Chek this out:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = ["<h1>Demo page</h1>"];
var pushHTML = html.push.bind(html);
Queue.push( getSomeData.partial(client, pushHTML) );
Queue.push( getSomeOtherData.partial(client, pushHTML) );
Queue.push( getMoreData.partial(client, pushHTML) );
Queue.push( function() {
res.write(html.join(''));
res.end();
});
Queue.execute();
});
Pretty neat, isn't it? You may notice that html became an array. That's partly because strings are immutable, so you better off with buffering your output in an array, than discarding larger and larger strings. The other reason is because of another nice syntax with bind.
Queue in the example is really just an example and along with partial can be implemented as follows
// Functional programming for the rescue
Function.prototype.partial = function() {
var fun = this,
preArgs = Array.prototype.slice.call(arguments);
return function() {
fun.apply(null, preArgs.concat.apply(preArgs, arguments));
};
};
Queue = [];
Queue.execute = function () {
if (Queue.length) {
Queue.shift()(Queue.execute);
}
};
Solution 7 - Javascript
Am in love Async.js ever since I found it. It has a async.series function you can use to avoid long nesting.
Documentation:-
series(tasks, [callback])
Run an array of functions in series, each one running once the previous function has completed. [...]
Arguments
tasks - An array of functions to run, each function is passed a callback it must call on completion.
callback(err, [results]) - An optional callback to run once all the functions have completed. This function gets an array of all the arguments passed to the callbacks used in the array.
Here's how we can apply it to your example code:-
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
async.series([
function (callback) {
getSomeData(client, function (someData) {
html += "<p>"+ someData +"</p>";
callback();
});
},
function (callback) {
getSomeOtherData(client, function (someOtherData) {
html += "<p>"+ someOtherData +"</p>";
callback();
});
},
funciton (callback) {
getMoreData(client, function (moreData) {
html += "<p>"+ moreData +"</p>";
callback();
});
}
], function () {
res.write(html);
res.end();
});
});
Solution 8 - Javascript
The most simple syntactical sugar i have seen is node-promise.
npm install node-promise || git clone https://github.com/kriszyp/node-promise
Using this you can chain async methods as:
firstMethod().then(secondMethod).then(thirdMethod);
The return value of each is available as argument in the next.
Solution 9 - Javascript
What you have done there is take an asynch pattern and apply it to 3 functions called in sequence, each one waiting for the previous one to complete before starting - i.e. you have made them synchronous. The point about asynch programming is that you can have several functions all running at once and not have to wait for each to complete.
if getSomeDate() doesn't provide anything to getSomeOtherDate(), which doesn't provide anything to getMoreData() then why don't you call them asynchronously as js allows or if they are interdependent (and not asynchronous) write them as a single function?
You don't need to use nesting to control the flow - for instance, get each function to finish by calling a common function that determines when all 3 have completed and then sends the response.
Solution 10 - Javascript
Suppose you could do this:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
chain([
function (next) {
getSomeDate(client, next);
},
function (next, someData) {
html += "<p>"+ someData +"</p>";
getSomeOtherDate(client, next);
},
function (next, someOtherData) {
html += "<p>"+ someOtherData +"</p>";
getMoreData(client, next);
},
function (next, moreData) {
html += "<p>"+ moreData +"</p>";
res.write(html);
res.end();
}
]);
});
You only need to implement chain() so that it partially applies each function to the next one, and immediately invokes only the first function:
function chain(fs) {
var f = function () {};
for (var i = fs.length - 1; i >= 0; i--) {
f = fs[i].partial(f);
}
f();
}
Solution 11 - Javascript
callback hell can be easily avoided in pure javascript with closure. the solution below assumes all callbacks follow the function(error, data) signature.
http.createServer(function (req, res) {
var modeNext, onNext;
// closure variable to keep track of next-callback-state
modeNext = 0;
// next-callback-handler
onNext = function (error, data) {
if (error) {
modeNext = Infinity;
} else {
modeNext += 1;
}
switch (modeNext) {
case 0:
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
getSomeDate(client, onNext);
break;
// handle someData
case 1:
html += "<p>"+ data +"</p>";
getSomeOtherDate(client, onNext);
break;
// handle someOtherData
case 2:
html += "<p>"+ data +"</p>";
getMoreData(client, onNext);
break;
// handle moreData
case 3:
html += "<p>"+ data +"</p>";
res.write(html);
res.end();
break;
// general catch-all error-handler
default:
res.statusCode = 500;
res.end(error.message + '\n' + error.stack);
}
};
onNext();
});
Solution 12 - Javascript
I've recently created simpler abstraction called wait.for to call async functions in sync mode (based on Fibers). It's at an early stage but works. It is at:
https://github.com/luciotato/waitfor
Using wait.for, you can call any standard nodejs async function, as if it were a sync function.
using wait.for your code could be:
var http=require('http');
var wait=require('wait.for');
http.createServer(function(req, res) {
wait.launchFiber(handleRequest,req, res); //run in a Fiber, keep node spinning
}).listen(8080);
//in a fiber
function handleRequest(req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
var someData = wait.for(getSomeDate,client);
html += "<p>"+ someData +"</p>";
var someOtherData = wait.for(getSomeOtherDate,client);
html += "<p>"+ someOtherData +"</p>";
var moreData = wait.for(getMoreData,client);
html += "<p>"+ moreData +"</p>";
res.write(html);
res.end();
};
...or if you want to be less verbose (and also add error catching)
//in a fiber
function handleRequest(req, res) {
try {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write(
"<h1>Demo page</h1>"
+ "<p>"+ wait.for(getSomeDate,client) +"</p>"
+ "<p>"+ wait.for(getSomeOtherDate,client) +"</p>"
+ "<p>"+ wait.for(getMoreData,client) +"</p>"
);
res.end();
}
catch(err) {
res.end('error '+e.message);
}
};
In all the cases, getSomeDate, getSomeOtherDate and getMoreData should be standard async functions with the last parameter a function callback(err,data)
as in:
function getMoreData(client, callback){
db.execute('select moredata from thedata where client_id=?',[client.id],
,function(err,data){
if (err) callback(err);
callback (null,data);
});
}
Solution 13 - Javascript
To solve this problem I wrote nodent (https://npmjs.org/package/nodent) which invisibly pre-processes your JS. Your example code would become (async, really - read the docs).
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
someData <<= getSomeDate(client) ;
html += "<p>"+ someData +"</p>";
someOtherData <<= getSomeOtherDate(client) ;
html += "<p>"+ someOtherData +"</p>";
moreData <<= getMoreData(client) ;
html += "<p>"+ moreData +"</p>";
res.write(html);
res.end();
});
Clearly, there are many other solutions, but pre-processing has the advantage of having little or no run-time overhead and thanks to source-map support it's easy to debug too.
Solution 14 - Javascript
I had the same problem. I've seen the major libs to node run async functions, and they presents so non-natural chaining (you need to use three or more methods confs etc) to build your code.
I spent some weeks developing a solution to be simple and easing to read. Please, give a try to EnqJS. All opinions will be appreciated.
Instead of:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
getSomeDate(client, function(someData) {
html += "<p>"+ someData +"</p>";
getSomeOtherDate(client, function(someOtherData) {
html += "<p>"+ someOtherData +"</p>";
getMoreData(client, function(moreData) {
html += "<p>"+ moreData +"</p>";
res.write(html);
res.end();
});
});
});
with EnqJS:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var html = "<h1>Demo page</h1>";
enq(function(){
var self=this;
getSomeDate(client, function(someData){
html += "<p>"+ someData +"</p>";
self.return();
})
})(function(){
var self=this;
getSomeOtherDate(client, function(someOtherData){
html += "<p>"+ someOtherData +"</p>";
self.return();
})
})(function(){
var self=this;
getMoreData(client, function(moreData) {
html += "<p>"+ moreData +"</p>";
self.return();
res.write(html);
res.end();
});
});
});
Observe that the code appears to be bigger than before. But it isn't nested as before. To appear more natural, the chains are called imediately:
enq(fn1)(fn2)(fn3)(fn4)(fn4)(...)
And to say that it returned, inside the function we call:
this.return(response)
Solution 15 - Javascript
I do it in a pretty primitive but effective way. E.g. I need to get a model with its parents and children and let's say I need to do separate queries for them:
var getWithParents = function(id, next) {
var getChildren = function(model, next) {
/*... code ... */
return next.pop()(model, next);
},
getParents = function(model, next) {
/*... code ... */
return next.pop()(model, next);
}
getModel = function(id, next) {
/*... code ... */
if (model) {
// return next callbacl
return next.pop()(model, next);
} else {
// return last callback
return next.shift()(null, next);
}
}
return getModel(id, [getParents, getChildren, next]);
}
Solution 16 - Javascript
Use Fibers https://github.com/laverdet/node-fibers it makes asynchronous code looks like synchronous (without blocking)
I personally use this little wrapper http://alexeypetrushin.github.com/synchronize Sample of code from my project (every method is actually asynchronous, working with async file IO) I even afraid of imagine what a mess it would be with callback or async-control-flow helper libraries.
_update: (version, changesBasePath, changes, oldSite) ->
@log 'updating...'
@_updateIndex version, changes
@_updateFiles version, changesBasePath, changes
@_updateFilesIndexes version, changes
configChanged = @_updateConfig version, changes
@_updateModules version, changes, oldSite, configChanged
@_saveIndex version
@log "updated to #{version} version"
Solution 17 - Javascript
Task.js offers you this:
spawn(function*() {
try {
var [foo, bar] = yield join(read("foo.json"),
read("bar.json")).timeout(1000);
render(foo);
render(bar);
} catch (e) {
console.log("read failed: " + e);
}
});
Instead of this:
var foo, bar;
var tid = setTimeout(function() { failure(new Error("timed out")) }, 1000);
var xhr1 = makeXHR("foo.json",
function(txt) { foo = txt; success() },
function(err) { failure() });
var xhr2 = makeXHR("bar.json",
function(txt) { bar = txt; success() },
function(e) { failure(e) });
function success() {
if (typeof foo === "string" && typeof bar === "string") {
cancelTimeout(tid);
xhr1 = xhr2 = null;
render(foo);
render(bar);
}
}
function failure(e) {
xhr1 && xhr1.abort();
xhr1 = null;
xhr2 && xhr2.abort();
xhr2 = null;
console.log("read failed: " + e);
}
Solution 18 - Javascript
After the others responded, you stated that your problem were local variables. It seems an easy way to do this is to write one outer function to contain those local variables, then use a bunch of named inner functions and access them by name. This way, you'll only ever nest two deep, regardless of how many functions you need to chain together.
Here is my newbie's attempt at using the mysql Node.js module with nesting:
function with_connection(sql, bindings, cb) {
pool.getConnection(function(err, conn) {
if (err) {
console.log("Error in with_connection (getConnection): " + JSON.stringify(err));
cb(true);
return;
}
conn.query(sql, bindings, function(err, results) {
if (err) {
console.log("Error in with_connection (query): " + JSON.stringify(err));
cb(true);
return;
}
console.log("with_connection results: " + JSON.stringify(results));
cb(false, results);
});
});
}
The following is a rewrite using named inner functions. The outer function with_connection can be used as a holder for local variables, too. (Here, I've got the parameters sql, bindings, cb that act in a similar way, but you can just define some additional local variables in with_connection.)
function with_connection(sql, bindings, cb) {
function getConnectionCb(err, conn) {
if (err) {
console.log("Error in with_connection/getConnectionCb: " + JSON.stringify(err));
cb(true);
return;
}
conn.query(sql, bindings, queryCb);
}
function queryCb(err, results) {
if (err) {
console.log("Error in with_connection/queryCb: " + JSON.stringify(err));
cb(true);
return;
}
cb(false, results);
}
pool.getConnection(getConnectionCb);
}
I had been thinking that perhaps it would be possible to make an object with instance variables, and to use these instance variables as a replacement for the local variables. But now I find that the above approach using nested functions and local variables is simpler and easier to understand. It takes some time to unlearn OO, it seems :-)
So here is my previous version with an object and instance variables.
function DbConnection(sql, bindings, cb) {
this.sql = sql;
this.bindings = bindings;
this.cb = cb;
}
DbConnection.prototype.getConnection = function(err, conn) {
var self = this;
if (err) {
console.log("Error in DbConnection.getConnection: " + JSON.stringify(err));
this.cb(true);
return;
}
conn.query(this.sql, this.bindings, function(err, results) { self.query(err, results); });
}
DbConnection.prototype.query = function(err, results) {
var self = this;
if (err) {
console.log("Error in DbConnection.query: " + JSON.stringify(err));
self.cb(true);
return;
}
console.log("DbConnection results: " + JSON.stringify(results));
self.cb(false, results);
}
function with_connection(sql, bindings, cb) {
var dbc = new DbConnection(sql, bindings, cb);
pool.getConnection(function (err, conn) { dbc.getConnection(err, conn); });
}
It turns out that bind can be used to some advantage. It allows me to get rid of the somewhat ugly anonymous functions I've created that didn't do anything much, except to forward themselves to a method call. I couldn't pass the method directly because it would have been involved with the wrong value of this. But with bind, I can specify the value of this that I want.
function DbConnection(sql, bindings, cb) {
this.sql = sql;
this.bindings = bindings;
this.cb = cb;
}
DbConnection.prototype.getConnection = function(err, conn) {
var f = this.query.bind(this);
if (err) {
console.log("Error in DbConnection.getConnection: " + JSON.stringify(err));
this.cb(true);
return;
}
conn.query(this.sql, this.bindings, f);
}
DbConnection.prototype.query = function(err, results) {
if (err) {
console.log("Error in DbConnection.query: " + JSON.stringify(err));
this.cb(true);
return;
}
console.log("DbConnection results: " + JSON.stringify(results));
this.cb(false, results);
}
// Get a connection from the pool, execute `sql` in it
// with the given `bindings`. Invoke `cb(true)` on error,
// invoke `cb(false, results)` on success. Here,
// `results` is an array of results from the query.
function with_connection(sql, bindings, cb) {
var dbc = new DbConnection(sql, bindings, cb);
var f = dbc.getConnection.bind(dbc);
pool.getConnection(f);
}
Of course, none of this is proper JS with Node.js coding -- I just spent a couple of hours on it. But maybe with a little polishing this technique can help?
Solution 19 - Javascript
async.js works well for this. I came across this very useful article which explains the need and use of async.js with examples: http://www.sebastianseilund.com/nodejs-async-in-practice
Solution 20 - Javascript
If you don't want to use "step" or "seq", please try "line" which is a simple function to reduce nested async callback.
Solution 21 - Javascript
C#-like asyncawait is another way of doing this
https://github.com/yortus/asyncawait
async(function(){
var foo = await(bar());
var foo2 = await(bar2());
var foo3 = await(bar2());
}
Solution 22 - Javascript
Using wire your code would look like this:
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var l = new Wire();
getSomeDate(client, l.branch('someData'));
getSomeOtherDate(client, l.branch('someOtherData'));
getMoreData(client, l.branch('moreData'));
l.success(function(r) {
res.write("<h1>Demo page</h1>"+
"<p>"+ r['someData'] +"</p>"+
"<p>"+ r['someOtherData'] +"</p>"+
"<p>"+ r['moreData'] +"</p>");
res.end();
});
});
Solution 23 - Javascript
for your know consider Jazz.js https://github.com/Javanile/Jazz.js/wiki/Script-showcase
const jj = require('jazz.js');
// ultra-compat stack
jj.script([
a => ProcessTaskOneCallbackAtEnd(a),
b => ProcessTaskTwoCallbackAtEnd(b),
c => ProcessTaskThreeCallbackAtEnd(c),
d => ProcessTaskFourCallbackAtEnd(d),
e => ProcessTaskFiveCallbackAtEnd(e),
]);