Sails.js populate nested associations

I've got myself a question regarding associations in Sails.js version 0.10-rc5. I've been building an app in which multiple models are associated to one another, and I've arrived at a point where I need to get to nest associations somehow.

There's three parts:

First there's something like a blog post, that's being written by a user. In the blog post I want to show the associated user's information like their username. Now, everything works fine here. Until the next step: I'm trying to show comments which are associated with the post.

The comments are a separate Model, called Comment. Each of which also has an author (user) associated with it. I can easily show a list of the Comments, although when I want to display the User's information associated with the comment, I can't figure out how to populate the Comment with the user's information.

In my controller i'm trying to do something like this:

Post
  .findOne(req.param('id'))
  .populate('user')
  .populate('comments') // I want to populate this comment with .populate('user') or something
  .exec(function(err, post) {
    // Handle errors & render view etc.
  });

In my Post's 'show' action i'm trying to retrieve the information like this (simplified):

<ul> 
  <%- _.each(post.comments, function(comment) { %>
    <li>
      <%= comment.user.name %>
      <%= comment.description %>
    </li>
  <% }); %>
</ul>

The comment.user.name will be undefined though. If I try to just access the 'user' property, like comment.user, it'll show it's ID. Which tells me it's not automatically populating the user's information to the comment when I associate the comment with another model.

Anyone any ideals to solve this properly :)?

Thanks in advance!

P.S.

For clarification, this is how i've basically set up the associations in different models:

// User.js
posts: {
  collection: 'post'
},   
hours: {
  collection: 'hour'
},
comments: {
  collection: 'comment'
}

// Post.js
user: {
  model: 'user'
},
comments: {
  collection: 'comment',
  via: 'post'
}

// Comment.js
user: {
  model: 'user'
},
post: {
  model: 'post'
}

Or you can use the built-in Blue Bird Promise feature to make it. (Working on [email protected])

See the codes below:

var _ = require('lodash');

...

Post
  .findOne(req.param('id'))
  .populate('user')
  .populate('comments')
  .then(function(post) {
    var commentUsers = User.find({
        id: _.pluck(post.comments, 'user')
          //_.pluck: Retrieves the value of a 'user' property from all elements in the post.comments collection.
      })
      .then(function(commentUsers) {
        return commentUsers;
      });
    return [post, commentUsers];
  })
  .spread(function(post, commentUsers) {
    commentUsers = _.indexBy(commentUsers, 'id');
    //_.indexBy: Creates an object composed of keys generated from the results of running each element of the collection through the given callback. The corresponding value of each key is the last element responsible for generating the key
    post.comments = _.map(post.comments, function(comment) {
      comment.user = commentUsers[comment.user];
      return comment;
    });
    res.json(post);
  })
  .catch(function(err) {
    return res.serverError(err);
  });

Some explanation:

1. I'm using the Lo-Dash to deal with the arrays. For more details, please refer to the Official Doc
2. Notice the return values inside the first "then" function, those objects "[post, commentUsers]" inside the array are also "promise" objects. Which means that they didn't contain the value data when they first been executed, until they got the value. So that "spread" function will wait the acture value come and continue doing the rest stuffs.

At the moment, there's no built in way to populate nested associations. Your best bet is to use async to do a mapping:

async.auto({

    // First get the post  
    post: function(cb) {
        Post
           .findOne(req.param('id'))
           .populate('user')
           .populate('comments')
           .exec(cb);
    },

    // Then all of the comment users, using an "in" query by
    // setting "id" criteria to an array of user IDs
    commentUsers: ['post', function(cb, results) {
        User.find({id: _.pluck(results.post.comments, 'user')}).exec(cb);
    }],

    // Map the comment users to their comments
    map: ['commentUsers', function(cb, results) {
        // Index comment users by ID
        var commentUsers = _.indexBy(results.commentUsers, 'id');
        // Get a plain object version of post & comments
        var post = results.post.toObject();
        // Map users onto comments
        post.comments = post.comments.map(function(comment) {
            comment.user = commentUsers[comment.user];
            return comment;
        });
        return cb(null, post);
    }]

}, 
   // After all the async magic is finished, return the mapped result
   // (or an error if any occurred during the async block)
   function finish(err, results) {
       if (err) {return res.serverError(err);}
       return res.json(results.map);
   }
);

It's not as pretty as nested population (which is in the works, but probably not for v0.10), but on the bright side it's actually fairly efficient.

I created an NPM module for this called nested-pop. You can find it at the link below.

https://www.npmjs.com/package/nested-pop

Use it in the following way.

var nestedPop = require('nested-pop');

User.find()
.populate('dogs')
.then(function(users) {

    return nestedPop(users, {
        dogs: [
            'breed'
        ]
    }).then(function(users) {
        return users
    }).catch(function(err) {
        throw err;
    });

}).catch(function(err) {
    throw err;
);

Worth saying there's a pull request to add nested population: https://github.com/balderdashy/waterline/pull/1052

Pull request isn't merged at the moment but you can use it installing one directly with

npm i Atlantis-Software/waterline#deepPopulate

With it you can do something like .populate('user.comments ...)'.

 sails v0.11 doesn't support _.pluck and _.indexBy use sails.util.pluck and sails.util.indexBy instead.

async.auto({

     // First get the post  
    post: function(cb) {
        Post
           .findOne(req.param('id'))
           .populate('user')
           .populate('comments')
           .exec(cb);
    },

    // Then all of the comment users, using an "in" query by
    // setting "id" criteria to an array of user IDs
    commentUsers: ['post', function(cb, results) {
        User.find({id:sails.util.pluck(results.post.comments, 'user')}).exec(cb);
    }],

    // Map the comment users to their comments
    map: ['commentUsers', function(cb, results) {
        // Index comment users by ID
        var commentUsers = sails.util.indexBy(results.commentUsers, 'id');
        // Get a plain object version of post & comments
        var post = results.post.toObject();
        // Map users onto comments
        post.comments = post.comments.map(function(comment) {
            comment.user = commentUsers[comment.user];
            return comment;
        });
        return cb(null, post);
    }]

}, 
   // After all the async magic is finished, return the mapped result
   // (or an error if any occurred during the async block)
   function finish(err, results) {
       if (err) {return res.serverError(err);}
       return res.json(results.map);
   }
);

You could use async library which is very clean and simple to understand. For each comment related to a post you can populate many fields as you want with dedicated tasks, execute them in parallel and retrieve the results when all tasks are done. Finally, you only have to return the final result.

Post
        .findOne(req.param('id'))
        .populate('user')
        .populate('comments') // I want to populate this comment with .populate('user') or something
        .exec(function (err, post) {

            // populate each post in parallel
            async.each(post.comments, function (comment, callback) {
                
                // you can populate many elements or only one...
                var populateTasks = {
                    user: function (cb) {
                        User.findOne({ id: comment.user })
                            .exec(function (err, result) {
                                cb(err, result);
                            });
                    }
                }

                async.parallel(populateTasks, function (err, resultSet) {
                    if (err) { return next(err); }

                    post.comments = resultSet.user;
                    // finish
                    callback();
                });

            }, function (err) {// final callback
                if (err) { return next(err); }

                return res.json(post);
            });
        });

As of sailsjs 1.0 the "deep populate" pull request is still open, but the following async function solution looks elegant enough IMO:

const post = await Post
    .findOne({ id: req.param('id') })
    .populate('user')
    .populate('comments');
if (post && post.comments.length > 0) {
   const ids = post.comments.map(comment => comment.id);
   post.comments = await Comment
      .find({ id: commentId })
      .populate('user');
}

Granted this is an old question, but a much simpler solution would be to loop over the comments,replacing each comment's 'user' property (which is an id) with the user's full detail using async await.

async function getPost(postId){
   let post = await Post.findOne(postId).populate('user').populate('comments');
   for(let comment of post.comments){
       comment.user = await User.findOne({id:comment.user});
   }
   return post;
}

Hope this helps!

In case anyone is looking to do the same but for multiple posts, here's one way of doing it:

• find all user IDs in posts
• query all users in 1 go from DB
• update posts with those users

Given that same user can write multiple comments, we're making sure we're reusing those objects. Also we're only making 1 additional query (whereas if we'd do it for each post separately, that would be multiple queries).

await Post.find()
  .populate('comments')
  .then(async (posts) => {
    // Collect all comment user IDs
    const userIDs = posts.reduce((acc, curr) => {
      for (const comment of post.comments) {
        acc.add(comment.user);
      }
      return acc;
    }, new Set());

    // Get users
    const users = await User.find({ id: Array.from(userIDs) });
    const usersMap = users.reduce((acc, curr) => {
      acc[curr.id] = curr;
      return acc;
    }, {});

    // Assign users to comments
    for (const post of posts) {
      for (const comment of post.comments) {
        if (comment.user) {
          const userID = comment.user;
          comment.user = usersMap[userID];
        }
      }
    }

    return posts;
  });