Improve publication speed to client

help
1

Hi,
I have a collectionX on the server and a collectionY on the client.
CollectionY is populated/updated through a subscription.

CollectionX has observers and I update CollectionY using Meteor’s low level publication api (added/changed/removed).
The setup works fairly well on a development machine where I have 4500 docs that get pushed to the client. In production the pubsub/ddp system is extremely slow when initializing CollectionY.

Someone mentioned to me that a Meteor.call method can send the initial array of docs much faster to the client. However, I lose the ability to observe the changes to CollectionX and updating CollectionY without using Meteor.subscribe.

Does somebody have a pattern where you can quickly initialize a collection on the client and then update the client collection later as changes to the server collection changes? I would not like to resend the whole dataset to the client each time only 1 doc on the server is changed.

2

Hi,
Below shows my attempt to combine my (fast)method and (slow, but reactive)subscription inside a react container.
It loads the client collection, but when I test the reactivity by removing a doc(line) on the server (using the mongo console) I get the following error on the client:
Uncaught Error: Expected not to find a document already present for an add

//========== Client only code =============
export const Lines2 = new Mongo.Collection('Lines2');

function insertBulk(collection, documents) {
  if (collection) {
    const last = _.last(documents);
    _.each(documents, (item) => {
      if (_.isObject(item)) {
        // console.log('bulked');
        if (false) console.log('false'); //(item._id === last._id) {console.log('a');collection.upsert({ _id: last._id }, item);}
        else
          if (_.isObject(item._id)) {
            const string = item._id.toHexString()
            collection._collection._docs._map[string] = item;
          } else collection._collection._docs._map[item._id] = item;
      }
    });
  }
}

function composer(props, onData) {
  const { modelId } = props;
  const publishedLines = {};
  Meteor.call('lines2.method', modelId, (err, lines) => {
    if (err) throw err;
    else {
      lines.forEach(line => {
        publishedLines[line._id._str] = true;
      });
      insertBulk(Lines2, lines);
    }
  });
  if (Meteor.subscribe('lines2.bymodelId', modelId, publishedLines).ready()) {
    console.log('sub ready');
    const lines = Lines2.find().fetch();
    onData(null, {
      lines: Immutable.fromJS(lines),
    });
  }
}

export const LinesContainer2 = composeWithTracker(composer, null, null, { withRef: true })(LinesLayer);
//==========Server only code===
function updatePub(self, simplifiedlinks, publishedLines) {
  const latestIds = simplifiedlinks.map((link) => link._id._str);
  const publishedIds = Object.keys(publishedLines);

  _.difference(publishedIds, latestIds).forEach((key) => { // _ needs strings not objectIds
    delete publishedLines[key];
    self.removed('Lines2', new Mongo.ObjectID(key));
  });

  simplifiedlinks.forEach((link) => {
    if (publishedLines[link._id._str]) {
      self.changed('Lines2', link._id, link);
    } else {
      publishedLines[link._id._str] = true;
      if (publishedLines[link._id._str]) {
        self.added('Lines2', link._id, link);
      }
    }
  });
}

Meteor.methods({
  'lines2.method': function(modelId){
    console.log('lines2.method');
    check(modelId, Mongo.ObjectID);
    const lines = calcLines(modelId, this.userId);
    return lines;
  },
});

Meteor.publish('lines2.bymodelId', function linesPublication(modelId, publishedLines) {
  console.log('lines2.bymodelId');
  check(modelId, Mongo.ObjectID);
  check(publishedLines, Object);
  const self = this;
  const model = Models.findOne(modelId);
  if (!model) self.ready();
  else if (!isMember.bind(self, model.modelId)) self.ready();
  else {
    // const publishedLines = {};
    let initializing = true;

    const linksHandle = Links.find({ modelId }).observe({ // update client if the links change
      added(newLink) {
        if (!initializing) {
          console.log('linksHandle added newLink ',newLink)
          const simplifiedlinks = calcLines(modelId, self.userId);
          updatePub(self, simplifiedlinks, publishedLines);
        }
      },
      changed(newLink, oldLink) {
        console.log('linksHandle changed newLink ',newLink,' oldLink',oldLink)
        const simplifiedlinks = calcLines(modelId, self.userId);
        updatePub(self, simplifiedlinks, publishedLines);
      },
      removed(oldLink) {
        console.log('linksHandle removed oldLink ',oldLink)
        const simplifiedlinks = calcLines(modelId, self.userId);
        updatePub(self, simplifiedlinks, publishedLines);
      },
    });


    initializing = false;
    // const simplifiedlinks = calcLines(modelId, memberId);
    // updatePub(self, simplifiedlinks, publishedLines);
    self.ready();

    self.onStop(() => {
      linestylesHandle.stop();
      linksHandle.stop();
    });
  }
});
3

Please use the key under escape instead of the key next to enter for the tripple apostrophe thing to properly display the code.

4

Apologies, corrected

1 Like
5

Hi, am I barking up the wrong tree with my combined method/subscription/bulkInsert approach?

6

Hi, it looks like what I am trying to do is to ‘seed’ the client side collection. I see that richsilv/meteor-dumb-collections does something like that. However, this package was last updated 2 years ago. Is there a more up to date method to seed a client collection and keep it tied to a publication with server side observers?

7

Looks like you simply don’t trust the reactivity…?
I might be wrong, but it seems to me that you are just adding complexity that turns into new bottle necks?