function (tId, partialTree) {
            if (tId === void 0) { tId = ''; }
            if (partialTree === void 0) { partialTree = tree; }
            if (!tId) {
                return undefined;
            }
            var found;
            partialTree.some(function (treeItem) {
                if (treeItem[id] === tId) {
                    found = treeItem;
                    return true;
                }
                found = treeItem[children] ? find(tId, treeItem[children]) : undefined;
                return found ? true : false;
            });
            return found;
        }

function ( tId , partialTree ) { if ( tId === void 0 ) { tId = '' ; } if ( partialTree === void 0 ) { partialTree = tree ; } if ( ! tId ) { return undefined ; } var found ; partialTree . some ( function ( treeItem ) { if ( treeItem [ id ] === tId ) { found = treeItem ; return true ; } found = treeItem [ children ] ? find ( tId , treeItem [ children ] ) : undefined ; return found ? true : false ; } ) ; return found ; }


function (el) {
        return el.id ? el.id : el.parentElement ? findId(el.parentElement) : null;
    }

function ( el ) { return el . id ? el . id : el . parentElement ? findId ( el . parentElement ) : null ; }

Find the ID of the first parent element.

Find the ID of the first parent element .


function listLocalDatasetClientData(datasetClient, cb) {
  syncStorage.readDatasetClientWithRecords(datasetClient.getId(), function(err, datasetClientsWithRecords) {
    if (err) {
      return cb(err);
    }
    //no one sync loop has completed yet, return null
    if (!datasetClientsWithRecords || !datasetClientsWithRecords.syncCompleted) {
      return cb(null, null);
    } else {
      return cb(null, datasetClientsWithRecords);
    }
  });
}

List the records for the given datasetClient
@param {DatasetClient} datasetClient
@param {Function} cb


function listAppliedChangeSinceLastSync(datasetId, lastSyncEndTime, clientInfo, cb) {
  syncStorage.listUpdates(datasetId, {
    type: SYNC_UPDATE_TYPES.APPLIED,
    cuid: clientInfo.cuid,
    timestamp: {$gt: lastSyncEndTime}
  }, null, cb);
}

function listAppliedChangeSinceLastSync ( datasetId , lastSyncEndTime , clientInfo , cb ) { syncStorage . listUpdates ( datasetId , { type : SYNC_UPDATE_TYPES . APPLIED , cuid : clientInfo . cuid , timestamp : { $gt : lastSyncEndTime } } , null , cb ) ; }

List the applied changes since the last sync loop for the given client
@param {String} datasetId the id of the dataset
@param {Number} lastSyncEndTime the timestamp of when the last sync loop completed
@param {Object} clientInfo the client info. Should contain the cuid
@param {Function} cb


function listPendingChangesForClient(datasetId, clientInfo, cb) {
  pendingQueue.search({datasetId: datasetId, cuid: clientInfo.cuid}, cb);
}

function listPendingChangesForClient ( datasetId , clientInfo , cb ) { pendingQueue . search ( { datasetId : datasetId , cuid : clientInfo . cuid } , cb ) ; }

List the pending changes for the given client that are still waiting to be processed
@param {String} datasetId
@param {Object} clientInfo the client info. Should contain the cuid
@param {Function} cb


function removePendingChanges(clientRecords, localDatasetClient, pendingChanges) {
  _.each(pendingChanges, function(pendingChange, uid) {
    if (clientRecords[uid]) {
      delete clientRecords[uid];
    }
    if (localDatasetClient[uid]) {
      delete localDatasetClient[uid];
    }
  });
}

function removePendingChanges ( clientRecords , localDatasetClient , pendingChanges ) { _ . each ( pendingChanges , function ( pendingChange , uid ) { if ( clientRecords [ uid ] ) { delete clientRecords [ uid ] ; } if ( localDatasetClient [ uid ] ) { delete localDatasetClient [ uid ] ; } } ) ; }

We have some pending changes sent in by the client and waiting in the queue. They are not processed yet so the backend or our localDataset don't have those changes.
We don't know if those changes will be applied or not. But they are already applied on the client which means if we compute the delta as it is, those changes will be reverted.
To avoid that, we will remove those changes from both the client and the localDataset so that the client will keep them for now.
@param {any} clientRecordds
@param {any} localDatasetClient
@param {any} pendingChanges

We have some pending changes sent in by the client and waiting in the queue . They are not processed yet so the backend or our localDataset don t have those changes . We don t know if those changes will be applied or not . But they are already applied on the client which means if we compute the delta as it is those changes will be reverted . To avoid that we will remove those changes from both the client and the localDataset so that the client will keep them for now .


function computeDelta(datasetId, clientRecords, serverRecords) {
  var creates = {};
  var updates = {};
  var deletes = {};
  _.each(serverRecords, function(serverRecord, serverRecordUid) {
    var serverRecHash = serverRecord.hash;
    //record is in both client and server
    if (clientRecords[serverRecordUid]) {
      //hash value doesn't match, needs update
      if (clientRecords[serverRecordUid] !== serverRecHash) {
        debug('[%s] Updating client record %s  client hash=%s', datasetId, serverRecordUid, clientRecords[serverRecordUid]);
        updates[serverRecordUid] = serverRecord;
      }
    } else {
      //record is not in the client, needs create
      debug('[%s] Creating client record %s', datasetId, serverRecordUid);
      creates[serverRecordUid] = serverRecord;
    }
  });

  _.each(clientRecords, function(clientRecordHash, clientRecordUid) {
    if (!serverRecords[clientRecordUid]) {
      //the record is in the client but not in the server, need delete
      debug('[%s] Deleting client record %s', datasetId, clientRecordUid);
      deletes[clientRecordUid] = {};
    }
  });

  return {
    create: creates,
    update: updates,
    delete: deletes
  };
}

Find out the differences between the clientRecords and the serverRecords
@param {String} datasetId the id of the dataset
@param {Object} clientRecords client records. The key should be the uid of each record, and the value should be the hash of each record
@param {Object} serverRecords server records. The key should be the uid of each record, and the value should contain the `data` and the `hash` of each record
@returns the records that should be created/updated/deleted from the client


function syncRecords(datasetId, params, cb) {
  debug('[%s] process syncRecords request', datasetId);
  var queryParams = params.query_params || {};
  var metaData = params.meta_data || {}; //NOTE: the client doesn't send in this value for syncRecords ATM
  var cuid = syncConfig.cuidProducer(params);
  var datasetClient = new DatasetClient(datasetId, {queryParams: queryParams, metaData: metaData});
  var clientRecords = params.clientRecs || {};
  async.waterfall([
    function checkDatasetclientStopped(callback) {
      syncStorage.readDatasetClient(datasetClient.getId(), function(err, datasetClientJson) {
        if (err) {
          return callback(err);
        }
        if (!datasetClientJson) {
          var errMsg = "unknown dataset client datasetId = " + datasetId + " :: queryParams = " + util.inspect(queryParams);
          debugError("[%s] %s", datasetId, errMsg);
          return callback(errMsg);
        }
        if (datasetClientJson.stopped === true) {
          return callback(new Error('sync stopped for dataset ' + datasetId));
        } else {
          return callback();
        }
      });
    },
    async.apply(listLocalDatasetClientData, datasetClient),
    function listOtherChanges(localDatasetData, callback) {
      if (localDatasetData) {
        listChangesNotInLocalDataset(datasetId, localDatasetData.syncLoopEnd, {cuid: cuid}, function(err, appliedUpdates, clientPendingChanges) {
          return callback(err, localDatasetData, appliedUpdates, clientPendingChanges);
        });
      } else {
        return callback();
      }
    }
  ], function(err, localDatasetData, appliedUpdatesSinceLastSync, pendingUpdates) {
    if (err) {
      return cb(err);
    }
    if (!localDatasetData) {
      return cb(null, {});
    }
    var localDatasetDataObj = convertToObject(localDatasetData.records);
    var appliedUpdatesSinceLastSyncObj = convertToObject(appliedUpdatesSinceLastSync);
    var pendingUpdatesObj = convertToObject(pendingUpdates);

    removeAppliedUpdates(clientRecords, localDatasetDataObj, appliedUpdatesSinceLastSyncObj);
    removePendingChanges(clientRecords, localDatasetDataObj, pendingUpdatesObj);

    var delta = computeDelta(datasetId, clientRecords, localDatasetDataObj);
    var res = delta;
    res.hash = localDatasetData.globalHash;
    debug('[%s] syncRecords API response %j', datasetId, res);
    return cb(null, res);
  });
}

Sync the data records for the given client
@param {String} datasetId
@param {Object} params the request body, it normally contain those fields:
@param {Object} params.query_params the query parameter for the dataset from the client
@param {Object} params.meta_data the meta data for the dataset from the client
@param {Object} params.__fh an object added by the client sdk. it should have the `cuid` of the client
@param {Object} params.clientRecs the client records
@param {Function} cb


function(metric, records) {
  var returnValue = {message: 'no stats available', name: metric.displayName};
  if (records && records.length > 0) {
    returnValue = _.chain(records).map(function(recordStr) {
      return JSON.parse(recordStr);
    }).groupBy(function(record) {
      return record.tags[metric.groupByTag];
    }).reduce(function(reduced, groupRecords, groupKey) {
      groupRecords = _.sortBy(groupRecords, 'ts');
      var processedData = _.reduce(groupRecords, function(memo, groupRecord) {
        var value = groupRecord.fields[metric.valueField];
        memo.current = value;
        memo.numberOfRecords++;
        memo.total += value;
        memo.max = Math.max(value, memo.max);
        memo.min = Math.min(value, memo.min);
        memo.from = Math.min(groupRecord.ts, memo.from);
        memo.end = Math.max(groupRecord.ts, memo.end);
        return memo;
      }, {max: 0, min: MAX_NUMBER, current: 0, numberOfRecords: 0, total: 0, from: MAX_NUMBER, end: 0});
      reduced[groupKey] = {
        current: metric.dataFormatter(processedData.current),
        max: metric.dataFormatter(processedData.max),
        min: metric.dataFormatter(processedData.min),
        average: metric.dataFormatter(processedData.total / processedData.numberOfRecords),
        numberOfRecords: processedData.numberOfRecords,
        from: new Date(processedData.from).toISOString(),
        end: new Date(processedData.end).toISOString()
      };
      return reduced;
    }, {}).value();
  }
  return returnValue;
}

function
(
metric
,
records
)
{
var
returnValue
=
{
message
:
'no stats available'
,
name
:
metric
.
displayName
}
;
if
(
records
&&
records
.
length
>
0
)
{
returnValue
=
_
.
chain
(
records
)
.
map
(
function
(
recordStr
)
{
return
JSON
.
parse
(
recordStr
)
;
}
)
.
groupBy
(
function
(
record
)
{
return
record
.
tags
[
metric
.
groupByTag
]
;
}
)
.
reduce
(
function
(
reduced
,
groupRecords
,
groupKey
)
{
groupRecords
=
_
.
sortBy
(
groupRecords
,
'ts'
)
;
var
processedData
=
_
.
reduce
(
groupRecords
,
function
(
memo
,
groupRecord
)
{
var
value
=
groupRecord
.
fields
[
metric
.
valueField
]
;
memo
.
current
=
value
;
memo
.
numberOfRecords
++
;
memo
.
total
+=
value
;
memo
.
max
=
Math
.
max
(
value
,
memo
.
max
)
;
memo
.
min
=
Math
.
min
(
value
,
memo
.
min
)
;
memo
.
from
=
Math
.
min
(
groupRecord
.
ts
,
memo
.
from
)
;
memo
.
end
=
Math
.
max
(
groupRecord
.
ts
,
memo
.
end
)
;
return
memo
;
}
,
{
max
:
0
,
min
:
MAX_NUMBER
,
current
:
0
,
numberOfRecords
:
0
,
total
:
0
,
from
:
MAX_NUMBER
,
end
:
0
}
)
;
reduced
[
groupKey
]
=
{
current
:
metric
.
dataFormatter
(
processedData
.
current
)
,
max
:
metric
.
dataFormatter
(
processedData
.
max
)
,
min
:
metric
.
dataFormatter
(
processedData
.
min
)
,
average
:
metric
.
dataFormatter
(
processedData
.
total
/
processedData
.
numberOfRecords
)
,
numberOfRecords
:
processedData
.
numberOfRecords
,
from
:
new
Date
(
processedData
.
from
)
.
toISOString
(
)
,
end
:
new
Date
(
processedData
.
end
)
.
toISOString
(
)
}
;
return
reduced
;
}
,
{
}
)
.
value
(
)
;
}
return
returnValue
;
}

Compute the max, min, current, average values from the records
@param {Object} metric some meta data about the metric
@param {Array} records

Compute the max min current average values from the records


function init(dataset_id, options, cb) {
  debug('[%s] init sync with options %j', dataset_id, options);
  datasets.init(dataset_id, options);
  //make sure we use the exported version here as the start function should be called only ONCE
  module.exports.api.start(function(err) {
    if (err) {
      return cb(err);
    }
    syncStorage.updateManyDatasetClients({datasetId: dataset_id}, {stopped: false}, cb);
  });
}

Initialise cloud data sync service for specified dataset.

Initialise cloud data sync service for specified dataset .


function stop(dataset_id, cb) {
  if (!syncStarted) {
    return cb();
  }
  debug('[%s] stop sync for dataset', dataset_id);
  syncStorage.updateManyDatasetClients({datasetId: dataset_id}, {stopped: true}, cb);
}

function
stop
(
dataset_id
,
cb
)
{
if
(
!
syncStarted
)
{
return
cb
(
)
;
}
debug
(
'[%s] stop sync for dataset'
,
dataset_id
)
;
syncStorage
.
updateManyDatasetClients
(
{
datasetId
:
dataset_id
}
,
{
stopped
:
true
}
,
cb
)
;
}

Stop cloud data sync for the specified dataset_id


function stopAll(cb) {
  //sync is not started yet, but connect could be called already. In this case, just reset a few things
  if (!syncStarted) {
    interceptors.restore();
    dataHandlers.restore();
    hashProvider.restore();
    mongoDbClient = null;
    redisClient = null;
    metricsClient = null;
    return cb();
  }
  debug('stopAll syncs');
  datasetClientCleaner.stop();
  async.parallel([
    async.apply(syncStorage.updateManyDatasetClients, {}, {stopped: true}),
    async.apply(stopAllWorkers, syncWorkers),
    async.apply(stopAllWorkers, ackWorkers),
    async.apply(stopAllWorkers, pendingWorkers),
    async.apply(syncScheduler.stop.bind(syncScheduler))
  ], function(err) {
    if (err) {
      debugError('Failed to stop sync due to error : %s', err);
      return cb(err);
    }
    setConfig();
    interceptors.restore();
    dataHandlers.restore();
    hashProvider.restore();
    mongoDbClient = null;
    redisClient = null;
    metricsClient = null;
    ackQueue = null;
    pendingQueue = null;
    syncQueue = null;
    ackWorkers = [];
    pendingWorkers = [];
    syncWorkers = [];
    syncStarted = false;
    syncLock = null;
    datasetClientCleaner = null;
    // Reset the memoized start fn so it can be called again
    module.exports.api.start = async.memoize(start);
    return cb();
  });
}

Stop cloud data sync service for ALL datasets and reset.
This should really only used by tests.

Stop cloud data sync service for ALL datasets and reset . This should really only used by tests .


function removeCollision(datasetId, params, cb) {
  debug('[%s] removeCollision');
  dataHandlers.removeCollision(datasetId, params.hash, params.meta_data, cb);
}

function
removeCollision
(
datasetId
,
params
,
cb
)
{
debug
(
'[%s] removeCollision'
)
;
dataHandlers
.
removeCollision
(
datasetId
,
params
.
hash
,
params
.
meta_data
,
cb
)
;
}

Defines a handler function for deleting a collision from the collisions list.
Should be called after the dataset is initialised.

Defines a handler function for deleting a collision from the collisions list . Should be called after the dataset is initialised .


function doListDatasetClients(filter, cb) {
  debug('doListDatasetClients');
  var col = mongoClient.collection(DATASETCLIENTS_COLLECTION);
  col.find(filter).toArray(function(err, datasetClients) {
    if (err) {
      debugError('Failed to list datasetClients due to error %s', err);
    }
    return cb(err, datasetClients);
  });
}

The dataset client record object
@typedef {Object} storage~DatasetClientRecord
@property {String} uid - the uid of the record
@property {String} hash - the hash value of the data field of the record
@property {Object} data - the actual data of the record

List all the dataset clients docs.
@param {Object} filter
@param {Function} cb


function doRemoveDatasetClients(datasetClientsToRemove, cb) {
  var removeIds = _.pluck(datasetClientsToRemove, 'id');
  var datasetIds = _.uniq(_.pluck(datasetClientsToRemove, 'datasetId'));
  debug('doRemoveDatasetClients: removed datasetClients = %d, datasets = %d', removeIds.length, datasetIds.length);
  async.series([
    function deleteDatasetClientAndRefs(callback) {
      async.map(removeIds, doRemoveDatasetClientWithRecords, function(err, deleteResult) {
        if (err) {
          debug('Failed to delete datasetClients due to error %s', err);
        }
        return callback(err, deleteResult);
      });
    },
    function removeUnusedRecords(callback) {
      async.map(datasetIds, removeRecordsForDataset, function(err, deletedCount) {
        if (err) {
          debug('Error occured when delete records for dataset due to error %s', err);
        }
        return callback(err, deletedCount);
      });
    }
  ], function(err, results){
    if (err) {
      return cb(err);
    } else {
      return cb(null, results[0]);
    }
  });
}

Remove the given datasetClients
@param {storage~DatasetClient[]} datasetClientsToRemove an array of datasetClients to remove. Each of them should have an `id` field
@param {Function} cb


function doRemoveDatasetClientWithRecords(datasetClientId, cb) {
  debug('doRemoveDatasetClientWithRecords datasetClientId = %s',  datasetClientId);
  async.waterfall([
    async.apply(doReadDatasetClient, datasetClientId),
    function removeRefs(datasetClientJson, next) {
      var datasetId = datasetClientJson.datasetId;
      var recordsCollection = mongoClient.collection(getDatasetRecordsCollectionName(datasetId));
      recordsCollection.updateMany({'refs': datasetClientId}, {'$pull': {'refs': datasetClientId}}, function(err) {
        return next(err, datasetClientJson);
      });
    },
    function deleteDatasetClient(datasetClientJson, next) {
      var col = mongoClient.collection(DATASETCLIENTS_COLLECTION);
      col.findOneAndDelete({'id': datasetClientId}, next);
    }
  ], function(err, result) {
    if (err) {
      debug('Failed to doRemoveDatasetClientWithRecords due to error %s', err);
      return cb(err);
    } else {
      return cb(null, result && result.value);
    }
  });
}

Remove the datasetClient that matches the given datasetClientId, and also remove the references of the datasetClient from the records.
@param {String} datasetClientId
@param {Function} cb

Remove the datasetClient that matches the given datasetClientId and also remove the references of the datasetClient from the records .


function doReadDatasetClient(datasetClientId, cb) {
  debug('doReadDatasetClient datasetClientId = %s', datasetClientId);
  var col = mongoClient.collection(DATASETCLIENTS_COLLECTION);
  col.findOne({id: datasetClientId}, function(err, datasetClient) {
    if (err) {
      debugError('Failed to read datasetClient due to error %s :: datasetClientId = %s', err, datasetClientId);
    }
    return cb(err, datasetClient);
  });
}

Read the dataset client from db
@param {String} datasetClientId the id of the datasetClient
@param {Function} cb


function doUpdateDatasetClient(datasetClientId, fields, upsert, cb) {
  debug('doUpdateDatasetClient datasetClientId = %s :: fields = %j', datasetClientId, fields);
  var col = mongoClient.collection(DATASETCLIENTS_COLLECTION);
  delete fields._id;
  col.findOneAndUpdate({id: datasetClientId}, {'$set': fields}, {upsert: upsert, returnOriginal: false}, function(err, result) {
    if (err) {
      debugError('Failed to update datasetClients due to error %s :: datasetClientId = %s :: fields = %j',err,datasetClientId,fields);
      return cb(err);
    }

    if (result.value === null) {
      return cb(new Error('DatasetClient not found for id ' + datasetClientId));
    }

    //ensure the indexes are create for a given dataset
    ensureIndexesForDataset(result.value.datasetId);
    return cb(null, result.value);
  });
}

Update the dataset client
@param {String} datasetClientId the datasetClient id
@param {storage~DatasetClient} fields the fields that need to be updated
@param {Boolean} upsert if true, the record will be created if can't be found
@param {Function} cb


function createIndexForCollection(collectionName, indexField, indexOpts) {
  var collection = mongoClient.collection(collectionName);
  collection.createIndex(indexField, indexOpts, function(err) {
    if (err) {
      debugError('Failed to create index for collection. collection = %s :: index = %j :: error = %j',collectionName,indexField,err);
    } else {
      debug('Index created for collection. Collection = %s  :: index = %j',collectionName,indexField);
    }
  });
}

Create an index on the given collection
@param {String} collectionName
@param {Object} indexField see mongodb indexes
@param {Object} indexOpts see mongodb indexes options


function ensureIndexesForDataset(datasetId) {
  createIndexForCollection(getDatasetRecordsCollectionName(datasetId), {'uid': 1}, {});
  createIndexForCollection(getDatasetRecordsCollectionName(datasetId), {'refs': 1}, {});
  createIndexForCollection(require('./sync-updates').getDatasetUpdatesCollectionName(datasetId), {'cuid': 1, 'hash': 1}, {});
}

function ensureIndexesForDataset ( datasetId ) { createIndexForCollection ( getDatasetRecordsCollectionName ( datasetId ) , { 'uid' : 1 } , { } ) ; createIndexForCollection ( getDatasetRecordsCollectionName ( datasetId ) , { 'refs' : 1 } , { } ) ; createIndexForCollection ( require ( './sync-updates' ) . getDatasetUpdatesCollectionName ( datasetId ) , { 'cuid' : 1 , 'hash' : 1 } , { } ) ; }

Create all the indexes on the collections for a given dataset
@param {String} datasetId


function diffRecords(localRecords, newRecords) {
  var recordsDiff = {};
  _.each(newRecords, function(record, uid) {
    if (localRecords[uid]) {
      if (localRecords[uid].hash !== record.hash) {
        record.op = "update";
        recordsDiff[uid] = record;
      }
    } else {
      record.op = "update";
      recordsDiff[uid] = record;
    }
  });
  _.each(localRecords, function(record, uid) {
    if (!newRecords[uid]) {
      record.op = "delete";
      recordsDiff[uid] = record;
    }
  });
  return recordsDiff;
}

function diffRecords ( localRecords , newRecords ) { var recordsDiff = { } ; _ . each ( newRecords , function ( record , uid ) { if ( localRecords [ uid ] ) { if ( localRecords [ uid ] . hash !== record . hash ) { record . op = "update" ; recordsDiff [ uid ] = record ; } } else { record . op = "update" ; recordsDiff [ uid ] = record ; } } ) ; _ . each ( localRecords , function ( record , uid ) { if ( ! newRecords [ uid ] ) { record . op = "delete" ; recordsDiff [ uid ] = record ; } } ) ; return recordsDiff ; }

Compute the diff between the 2 objects based on the hash value
@param {Object} localRecords
@param {Object} newRecords
@returns an object that contains the diff


function diff(datasetClientJson, localRecords, next) {
      var recordsDiff = diffRecords(syncUtil.convertToObject(localRecords), syncUtil.convertToObject(records));
      return next(null, datasetClientJson, recordsDiff);
    }

function diff ( datasetClientJson , localRecords , next ) { var recordsDiff = diffRecords ( syncUtil . convertToObject ( localRecords ) , syncUtil . convertToObject ( records ) ) ; return next ( null , datasetClientJson , recordsDiff ) ; }

to reduce the number of db operations, we try to compute only the records that need to be changed here.

to reduce the number of db operations we try to compute only the records that need to be changed here .


function(filter, callback) {
      metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doListDatasetClients)(filter, callback);
    }

function ( filter , callback ) { metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doListDatasetClients ) ( filter , callback ) ; }

list the currently managed dataset clients
@param {Function} callback


function(datasetClientsToRemove, callback) {
      if (!datasetClientsToRemove.length) {
        return callback();
      }
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doRemoveDatasetClients)(datasetClientsToRemove, callback);
    }

function ( datasetClientsToRemove , callback ) { if ( ! datasetClientsToRemove . length ) { return callback ( ) ; } return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doRemoveDatasetClients ) ( datasetClientsToRemove , callback ) ; }

remove the given datasetClients from the storage. Each datasetclient should have at least the `id` field
@param {storage~DatasetClient[]} datasetClientsToRemove
@param {Function} callback

remove the given datasetClients from the storage . Each datasetclient should have at least the id field


function(datasetClientId, fieldsToUpdate, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doUpdateDatasetClient)(datasetClientId, fieldsToUpdate, false, callback);
    }

function ( datasetClientId , fieldsToUpdate , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doUpdateDatasetClient ) ( datasetClientId , fieldsToUpdate , false , callback ) ; }

update the given dataset client
@param {String} datasetClientId the id of the datasetClient
@param {storage~DatasetClient} fieldsToUpdate the fields to update
@param {Function} callback


function(datasetClientId, fields, callback) {
      fields.id = datasetClientId;
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doUpdateDatasetClient)(datasetClientId, fields, true, callback);
    }

function ( datasetClientId , fields , callback ) { fields . id = datasetClientId ; return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doUpdateDatasetClient ) ( datasetClientId , fields , true , callback ) ; }

Create the give dataset if it doesn't no exist, or update the existing one.
Should return the datasetClient instance in the callback
@param {String} datasetClientId the id of the datasetClient
@param {storage~DatasetClient} fields the fields to upsert
@param {Function} callback

Create the give dataset if it doesn t no exist or update the existing one . Should return the datasetClient instance in the callback


function(datasetClientId, fields, records, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doUpdateDatasetClientWithRecords)(datasetClientId, fields, records, callback);
    }

function ( datasetClientId , fields , records , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doUpdateDatasetClientWithRecords ) ( datasetClientId , fields , records , callback ) ; }

update the given dataset client with the given records
@param {String} datasetClientId the id of the datasetClient
@param {storage~DatasetClient} fields the fields to update
@param {storage~DatasetClientRecord[]} records an array of records to save
@param {Function} callback


function(datasetClientId, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doReadDatasetClientWithRecordsUseCache)(datasetClientId, callback);
    }

function ( datasetClientId , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doReadDatasetClientWithRecordsUseCache ) ( datasetClientId , callback ) ; }

Read the given datasetclient record, and its assoicated records
@param {String} datasetClientId
@param {Function} callback

Read the given datasetclient record and its assoicated records


function doFindAndDeleteUpdate(datasetId, acknowledgement, callback) {
  debug('[%s] doFindAndDeleteUpdate acknowledgement = %j',datasetId,acknowledgement);
  var updatesCollection = mongoClient.collection(getDatasetUpdatesCollectionName(datasetId));
  updatesCollection.findOneAndDelete({cuid: acknowledgement.cuid, hash: acknowledgement.hash}, function(err, result) {
    if (err) {
      debugError('[%s] Failed to doFindAndDeleteUpdate due to error %s :: acknowledgement = %j',datasetId,err,acknowledgement);
      return callback(err);
    }
    return callback(null, result.value);
  });
}

The acknowledgement object
@typedef {Object} storage~Acknowledgement
@property {String} type - if this updates has been applied successfully. Possible values: applied, failed and collisions.
@property {String} cuid - the unique client id that submitted that update
@property {String} action - the operation that the update is trying to perform. Possible values: create/update/delete
@property {String} hash - the hash value (can be considered as the unique id) of the update
@property {String} uid - the uid of the record the update is for
@property {String} msg - any message about the update. could be the error message if the update is failed
@property {Number} timestamp - when the update is applied

Find the delete the given sync update object from the `fhsync-<datasetId>-updates` collection
@param {String} datasetId
@param {storage~Acknowledgement} acknowledgement should at least have the `cuid` and `hash` fields
@param {Function} callback


function doListUpdates(datasetId, criteria, options, callback) {
  debug('[%s] doListUpdates criteria = %j',datasetId,criteria);
  var updatesCollection = mongoClient.collection(getDatasetUpdatesCollectionName(datasetId));
  var docLimit = options && options.limit;
  var cursor = updatesCollection.find(criteria);
  if (docLimit && docLimit > 0) {
    cursor = cursor.limit(docLimit);
  }
  cursor.toArray(function(err, updates) {
    if (err) {
      debugError('[%s] Failed to doListUpdates due to error %s :: criteria = %j' + criteria,datasetId,err,criteria);
      return callback(err);
    }
    return callback(null, updates);
  });
}

List all the updates that match the given search criteria

@param {String} datasetId
@param {Object} criteria
@param {Object} options
@param {Function} callback


function(datasetId, acknowledgement, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doFindAndDeleteUpdate)(datasetId, acknowledgement, callback);
    }

function ( datasetId , acknowledgement , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doFindAndDeleteUpdate ) ( datasetId , acknowledgement , callback ) ; }

find the delete the given sync update
@param {String} datasetId
@param {storage~Acknowledgement} acknowledgement
@param {Function} callback


function(datasetId, acknowledgementFields, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doSaveUpdate)(datasetId, acknowledgementFields, callback);
    }

function ( datasetId , acknowledgementFields , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doSaveUpdate ) ( datasetId , acknowledgementFields , callback ) ; }

Save the given sync update
@param {String} datasetId
@param {storage~Acknowledgement} acknowledgementFields
@param {Function} callback


function(datasetId, criteria, options, callback) {
      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doListUpdates)(datasetId, criteria, options, callback);
    }

function ( datasetId , criteria , options , callback ) { return metrics . timeAsyncFunc ( metrics . KEYS . MONGODB_OPERATION_TIME , doListUpdates ) ( datasetId , criteria , options , callback ) ; }

List the updates that match the given list criteria
@param {String} datasetId
@param {Object} criteria the list criteria, a mongodb query object
@param {Object} options options for the find option, like `limit`
@param {Function} callback


function DatasetClient(datasetId, opts){
  opts = opts || {};
  this.datasetId = datasetId;
  this.queryParams = opts.queryParams || {};
  this.metaData = opts.metaData || {};
  this.id = generateDatasetClientId(this);
  this.config = opts.config || datasets.getDatasetConfig(datasetId);
  this.collisionCount = opts.collisionCount || 0;
  this.stopped = opts.stopped;
  this.syncScheduled = opts.syncScheduled;
  this.syncCompleted = opts.syncCompleted;
  this.syncLoopStart = opts.syncLoopStart;
  this.syncLoopEnd = opts.syncLoopEnd;
  this.lastAccessed = opts.lastAccessed;
}

function DatasetClient ( datasetId , opts ) { opts = opts || { } ; this . datasetId = datasetId ; this . queryParams = opts . queryParams || { } ; this . metaData = opts . metaData || { } ; this . id = generateDatasetClientId ( this ) ; this . config = opts . config || datasets . getDatasetConfig ( datasetId ) ; this . collisionCount = opts . collisionCount || 0 ; this . stopped = opts . stopped ; this . syncScheduled = opts . syncScheduled ; this . syncCompleted = opts . syncCompleted ; this . syncLoopStart = opts . syncLoopStart ; this . syncLoopEnd = opts . syncLoopEnd ; this . lastAccessed = opts . lastAccessed ; }

Represent a dataset client that is managed by sync framework.
@param {String} datasetId the id of the dataset
@param {Object} opts options of the dataset client
@param {Object} opts.queryParams the query parameters of the dataset client. Will be passed to the data handler to list data.
@param {Object} opts.metaData the meta data of the dataset client. Will be passed to the data handler to list data.
@param {Object} opts.config the configuration option of the dataset client

Represent a dataset client that is managed by sync framework .


function invoke(dataset_id, params, callback) {
  debug('invoke');

  if (arguments.length < 3) throw new Error('invoke requires 3 arguments');

  // Verify that fn param has been passed
  if (!params || !params.fn) {
    var err = new Error('no fn parameter provided in params "' + util.inspect(params) + '"');
    debugError('[%s] warn %s %j', dataset_id, err, params);
    return callback(err, null);
  }

  var fn = params.fn;

  // Verify that fn param is valid
  if (invokeFunctions.indexOf(fn) < 0) {
    return callback(new Error('invalid fn parameter provided in params "' + fn + '"'), null);
  }

  // We can only continue if sync has connected to its dependencies i.e. mongo & redis
  if (!syncConnected) {
    return callback(new Error('Sync not connected'));
  }

  var fnHandler =  module.exports[fn] || server[fn] || server.api[fn];

  server.api.start(function() {
    return fnHandler(dataset_id, params, callback);
  });
}


function convertToObject(itemArr) {
  var obj = {};
  _.each(itemArr, function(item) {
    obj[item.uid] = item;
  });
  return obj;
}

function convertToObject ( itemArr ) { var obj = { } ; _ . each ( itemArr , function ( item ) { obj [ item . uid ] = item ; } ) ; return obj ; }

convert the given array to an object, use the `uid` field of each item as the key
@param {Array} itemArr
@returns an object

convert the given array to an object use the uid field of each item as the key


function SyncScheduler(syncQueueImpl, options) {
  if (!syncQueueImpl) {
    throw new Error('syncQueueImpl is required');
  }
  this.syncQueue = syncQueueImpl;
  options = options || {};
  this.syncSchedulerLockName = options.syncSchedulerLockName || 'locks:sync:SyncScheduler';
  this.timeBetweenChecks = options.timeBetweenChecks || 500;
  this.timeBeforeCrashAssumed = options.timeBeforeCrashAssumed || 20000;
  this.stopped = false;
}

function
SyncScheduler
(
syncQueueImpl
,
options
)
{
if
(
!
syncQueueImpl
)
{
throw
new
Error
(
'syncQueueImpl is required'
)
;
}
this
.
syncQueue
=
syncQueueImpl
;
options
=
options
||
{
}
;
this
.
syncSchedulerLockName
=
options
.
syncSchedulerLockName
||
'locks:sync:SyncScheduler'
;
this
.
timeBetweenChecks
=
options
.
timeBetweenChecks
||
500
;
this
.
timeBeforeCrashAssumed
=
options
.
timeBeforeCrashAssumed
||
20000
;
this
.
stopped
=
false
;
}

A scheduled job that will run to check the status of the datasets that are managed by the sync framework.
@param {MongodbQueue} syncQueueImpl an instance of the MongodbQueue that is used to push sync requests
@param {Object} options the configuration options for the scheduler
@param {String} options.syncSchedulerLockName the name of the sync scheduler lock. Default: "locks:sync:SyncScheduler".
@param {Number} options.timeBetweenChecks an interval that decides how often the scheduler should be running. In milliseconds. Default: 500.
@param {Number} options.timeBeforeCrashAssumed a value that decides how long the lock should be accquired for. In milliseconds. Default: 20000.

A scheduled job that will run to check the status of the datasets that are managed by the sync framework .


function updateDatasetClients(wcb) {
          var datasetClientIds = _.pluck(datasetClientsToSync, 'id');
          syncStorage.updateManyDatasetClients({id: {$in: datasetClientIds}}, {syncScheduled: Date.now()}, wcb);
        }

function updateDatasetClients ( wcb ) { var datasetClientIds = _ . pluck ( datasetClientsToSync , 'id' ) ; syncStorage . updateManyDatasetClients ( { id : { $in : datasetClientIds } } , { syncScheduled : Date . now ( ) } , wcb ) ; }

make sure each dataset client is marked to be scheduled


function insertDocsToDb(dburl, collectionName, docs, cb) {
  MongoClient.connect(dburl, function(err, db){
    if (err) {
      return cb(err);
    }
    var col = db.collection(collectionName);
    col.insertMany(docs, function(err, result){
      if (err) {
        return cb(err);
      }
      return cb(null, result);
    });
  });
}

function insertDocsToDb ( dburl , collectionName , docs , cb ) { MongoClient . connect ( dburl , function ( err , db ) { if ( err ) { return cb ( err ) ; } var col = db . collection ( collectionName ) ; col . insertMany ( docs , function ( err , result ) { if ( err ) { return cb ( err ) ; } return cb ( null , result ) ; } ) ; } ) ; }

Helper function to insert some documents to the given db and collection
@param {String} dburl
@param {String} collectionName
@param {Array} docs
@param {Function} cb


function set(key, value, cb) {
  if (!syncConfig.useCache || !redisClient) {
    return cb && cb();
  }
  return redisClient.set(key, value, cb);
}

function set ( key , value , cb ) { if ( ! syncConfig . useCache || ! redisClient ) { return cb && cb ( ) ; } return redisClient . set ( key , value , cb ) ; }

Save data to the cache
@param {String} key
@param {String} value
@param {Function} cb


function get(key, cb) {
  if (!syncConfig.useCache || !redisClient) {
    return cb && cb();
  }
  return redisClient.get(key, cb);
}

function get ( key , cb ) { if ( ! syncConfig . useCache || ! redisClient ) { return cb && cb ( ) ; } return redisClient . get ( key , cb ) ; }

Read data from cache
@param {String} key
@param {Function} cb


function del(key, cb) {
  if (!syncConfig.useCache || !redisClient) {
    return cb && cb();
  }
  return redisClient.del(key, cb);
}

function del ( key , cb ) { if ( ! syncConfig . useCache || ! redisClient ) { return cb && cb ( ) ; } return redisClient . del ( key , cb ) ; }

Delete data from cache
@param {String} key
@param {Function} cb


function syncWithBackend(payload, tries, callback) {
  var datasetClientId = payload.id;
  var datasetId = payload.datasetId;
  var startTime = payload.startTime;

  if (!datasetClientId || !datasetId) {
    recordProcessTime(startTime, false);
    debugError("no datasetId value found in sync request payload %j" ,payload);
    return callback();
  }

  if (tries > 1) {
    //the request is already run once, but for some reason is not acked, we just make sure it's completed and ack it
    markDatasetClientAsCompleted(datasetClientId, startTime, callback);
    return;
  }

  var queryParams = payload.queryParams || {};
  var metaData = payload.metaData || {};
  //we need to add this so that if this sync processor crashed before reaching the end, the scheduler will still be able to push a sync request
  var expectedTimeout = datasets.getDatasetConfig(datasetId).backendListTimeout * 1000 || 5*60*1000;

  async.waterfall([
    function setSyncStart(cb) {
      var syncLoopStartTime = Date.now();
      syncStorage.updateDatasetClient(datasetClientId, {syncLoopStart: syncLoopStartTime, syncLoopEnd: syncLoopStartTime + expectedTimeout}, function(err, datasetClient){
        return cb(err, datasetClient);
      });
    },
    function listData(datasetClient, cb) {
      listDataWithTimeout({datasetClientId: datasetClientId, datasetId: datasetId, queryParams: queryParams, metaData: metaData}, expectedTimeout, function(err, res) {
        return cb(err, datasetClient, res);
      });
    },
    function saveRecords(datasetClient, recordsWithHash, cb) {
      var toHash = _.pluck(recordsWithHash, 'hash');
      var globalHash = hashProvider.globalHash(datasetId, toHash);
      var globalHashWithCollisionCount = [globalHash, datasetClient.collisionCount].join('_');
      syncStorage.updateDatasetClientWithRecords(datasetClientId, {globalHash: globalHashWithCollisionCount}, recordsWithHash, cb);
    }
  ], function(err){
    if (err) {
      debugError("[%s] Error when sync data with backend. error = %s",datasetId,err);
    }

    markDatasetClientAsCompleted(datasetClientId, startTime, callback);
  });
}

Perform the sync operation
@param {Object} payload the payload of the sync request
@param {Object} payload.id the id of the dataset client
@param {String} payload.datasetId the id of the dataset that needs to be synced
@param {Object} payload.queryParams the query params that is associated with the dataset
@param {Object} payload.metaData the meta data that is associated with the dataset
@param {Timestamp} payload.startTime when the sync request is created
@param {Number} tries the number of tries of the request
@param {Function} callback


function doCreate(datasetId, pendingChange, callback) {
  var record = pendingChange.post;
  var metaData = pendingChange.meta_data;
  debug('[%s] CREATE Start data = %j', datasetId, record);
  dataHandlers.doCreate(datasetId, record, metaData, function(err, data) {
    if (err) {
      debugError('[%s] CREATE Failed - : err = %s', datasetId, err);
    } else {
      debug('[%s] CREATE Success - uid = %s', datasetId, data.uid);
      pendingChange.oldUid = pendingChange.uid;
      pendingChange.uid = data.uid;
    }
    return saveUpdate(datasetId, pendingChange, err ? SYNC_UPDATE_TYPES.FAILED : SYNC_UPDATE_TYPES.APPLIED, err ? util.inspect(err) : null, callback);
  });
}

function
doCreate
(
datasetId
,
pendingChange
,
callback
)
{
var
record
=
pendingChange
.
post
;
var
metaData
=
pendingChange
.
meta_data
;
debug
(
'[%s] CREATE Start data = %j'
,
datasetId
,
record
)
;
dataHandlers
.
doCreate
(
datasetId
,
record
,
metaData
,
function
(
err
,
data
)
{
if
(
err
)
{
debugError
(
'[%s] CREATE Failed - : err = %s'
,
datasetId
,
err
)
;
}
else
{
debug
(
'[%s] CREATE Success - uid = %s'
,
datasetId
,
data
.
uid
)
;
pendingChange
.
oldUid
=
pendingChange
.
uid
;
pendingChange
.
uid
=
data
.
uid
;
}
return
saveUpdate
(
datasetId
,
pendingChange
,
err
SYNC_UPDATE_TYPES
.
FAILED
:
SYNC_UPDATE_TYPES
.
APPLIED
,
err
util
.
inspect
(
err
)
:
,
callback
)
;
}
)
;
}

Create the given pending change in the backend.
It will always create the record in the backend.
@param {String} datasetId the dataset id
@param {Object} pendingChange the pending change. It should have the following fields
@param {String} pendingChange.datasetId the dataset id of the pending change
@param {Object} pendingChange.meta_data the meta_data of the dataset
@param {String} pendingChange.action the action of the pending change, should be "create"
@param {String} pendingChange.uid the unique id of the data record. This is a temp uid generated by the client and should be replaced by the real uid once the record is created.
@param {String} pendingChange.cuid the unique client id of the client device
@param {String} pendingChange.hash a unique id of the pending change, normally it is a hash value generated from the content of the pendingChange object
@param {Object} pendingChange.post the record after change
@param {Function} callback

Create the given pending change in the backend . It will always create the record in the backend .


function doDelete(datasetId, pendingChange, callback) {
  debug('[%s] DELETE Start', datasetId);
  var metaData = pendingChange.meta_data;
  var uid = pendingChange.uid;
  dataHandlers.doRead(datasetId, uid, metaData, function(err, data) {
    if (err) {
      debugError('READ for DELETE Failed - uid = %s : err = %s', datasetId, uid, err);
      return saveUpdate(datasetId, pendingChange, SYNC_UPDATE_TYPES.FAILED, util.inspect(err), callback);
    }
    debug('[%s] READ for DELETE Success', datasetId);
    debug('[%s] READ for DELETE Data : \n%j', datasetId, data);

    var preHash = hashProvider.recordHash(datasetId, pendingChange.pre);
    var dataHash = hashProvider.recordHash(datasetId, data);

    debug('[%s] DELETE Hash Check %s (client :: dataStore) = %s :: %s', datasetId, uid, preHash.dataHash);

    if (!dataHash) {
      //record has already been deleted
      debug('[%s] DELETE Already performed - uid=%s', datasetId, uid);
      return saveUpdate(datasetId, pendingChange, SYNC_UPDATE_TYPES.APPLIED, null, callback);
    }
    else {
      if (preHash === dataHash) {
        dataHandlers.doDelete(datasetId, uid, metaData, function(err) {
          if (err) {
            debugError('[%s] DELETE Failed - uid=%s : err = %s', datasetId, err);
          } else {
            debug('[%s] DELETE Success - uid=%s : hash = %s', datasetId, dataHash);
          }
          return saveUpdate(datasetId, pendingChange, err ? SYNC_UPDATE_TYPES.FAILED : SYNC_UPDATE_TYPES.APPLIED, err ? util.inspect(err) : null, callback);
        });
      } else {
        debug('[%s] DELETE COLLISION \n Pre record from client:\n%j\nCurrent record from data store:\n%j', datasetId, syncUtil.sortObject(pendingChange.pre), syncUtil.sortObject(data));
        handleCollision(datasetId, metaData, pendingChange, dataHash, function(err) {
          if (err) {
            debugError('[%s] Failed to save collision uid = %s : err = %s', datasetId, err);
          }
        });
        return saveUpdate(datasetId, pendingChange, SYNC_UPDATE_TYPES.COLLISION, null, callback);
      }
    }
  });
}

Delete the given pending change from the backend.
It will only delete the record if the hash value of the `pre` record matches the hash value of the record from backend.
Otherwise the change is either already applied, or a collision will be generated.
@param {String} datasetId the dataset id
@param {Object} pendingChange the pending change. It should have the following fields
@param {String} pendingChange.datasetId the dataset id of the pending change
@param {Object} pendingChange.meta_data the meta_data of the dataset
@param {String} pendingChange.action the action of the pending change, should be "delete"
@param {String} pendingChange.uid the unique id of the data record
@param {String} pendingChange.cuid the unique client id of the client device
@param {String} pendingChange.hash a unique id of the pending change, normally it is a hash value generated from the content of the pendingChange object
@param {Object} pendingChange.pre the record before change.
@param {Function} callback

Delete the given pending change from the backend . It will only delete the record if the hash value of the pre record matches the hash value of the record from backend . Otherwise the change is either already applied or a collision will be generated .


function applyPendingChange(pendingChange, tries, callback) {
  var datasetId = pendingChange.datasetId;
  if (!datasetId || !pendingChange.action || !pendingChange.uid || !pendingChange.cuid || !pendingChange.hash) {
    debugError("[%s] invalid pendingChange request dropped :: item = %j", datasetId, pendingChange);
    return callback();
  }
  debug('[%s] processPending :: item = %j', datasetId, pendingChange);
  if (tries > retryLimit) {
    //the pendingChange has been processed before. Mark it as failed
    debugError('[%s] processPending failed :: tries = %d  :: item = %j', datasetId, tries, pendingChange);
    return saveUpdate(datasetId, pendingChange, SYNC_UPDATE_TYPES.FAILED, "crashed", callback);
  }

  pendingChange.tries = tries;

  var action = pendingChange.action.toLowerCase();

  var timer = metrics.startTimer();

  function onComplete(err) {
    metricsClient.gauge(metrics.KEYS.PENDING_CHANGE_PROCESS_TIME, {success: !err, action: action}, timer.stop());
    return callback(err);
  }

  switch (action) {
    case "create":
      doCreate(datasetId, pendingChange, onComplete);
      break;
    case "update":
      doUpdate(datasetId, pendingChange, onComplete);
      break;
    case "delete":
      doDelete(datasetId, pendingChange, onComplete);
      break;
    default:
      debugError("[%s] invalid pendingChange request dropped :: item = %j", datasetId, pendingChange);
      return onComplete();
  }
}

apply the given pending change to the backend using the dataHandlers
@param {Object} pendingChange the pending change object
@param {String} pendingChange.datasetId the dataset id of the pending change
@param {Object} pendingChange.meta_data the meta_data of the dataset
@param {String} pendingChange.action the action of the pending change, should be one of "create", "update" or "delete"
@param {String} pendingChange.uid the unique id of the data record
@param {String} pendingChange.cuid the unique client id of the client device
@param {String} pendingChange.hash a unique id of the pending change, normally it is a hash value generated from the content of the pendingChange object
@param {Object} pendingChange.pre the record before change. Optional.
@param {Object} pendingChange.post the record after change. Optional.
@param {Number} tries a counter to record how many times the given pending change has been executed
@param {Function} callback the callback function
@returns


function MongodbQueue(name, metrics, lock, opts) {
  if (!name) {
    throw new Error('name is required to create a mongodb queue');
  }
  if (!opts || !opts.mongodb) {
    throw new Error('mongodb is not specified to create mongodb queue');
  }
  this.queueName = name;
  this.metrics = metrics;
  this.lock = lock;
  this.lockName = opts.lockName || ('lock:sync:' + this.queueName);
  this.lockTimeout = opts.lockTimeout || 10000;
  this.mongodb = opts.mongodb;
  this.queueOptions = {
    visibility: opts.visibility || 30,
    ttl: opts.queueMessagesTTL || 24*60*60
  };
  this.queue;
}

construct a new queue that is backed by mongodb.
@param {String} name the name of the queue.
@param {Object} metrics an instance of fh-component-metrics client
@param {Object} lock an instance of the lock service
@param {Object} opts configuration options for the queue
@param {Object} opts.mongodb an instance of the mongodb connection. This is required.
@param {Number} opts.visibility see https://github.com/chilts/mongodb-queue#visibility---message-visibility-window
@param {Object} opts.queueMessagesTTL The TTL (time to live) value for the messages on the queue. Default to 24 hours.

construct a new queue that is backed by mongodb .


function listIndexes(callback) {
            debug('[%s] list existing indexes', self.queueName);
            collection.indexInformation({full: true}, function(err, indexInfo) {
              if (err) {
                debug('[%s] error getting indexInfo. skipping ttl index check: %s', self.queueName, err);
                return callback(null, null);
              }
              return callback(null, indexInfo);
            });
          }

Check if there's already a TTL index for the 'deleted' field. If there is, and the TTL value is different, delete it so it can be recreated by the mongodb-queue module

Check if there s already a TTL index for the deleted field . If there is and the TTL value is different delete it so it can be recreated by the mongodb - queue module


function(datasetId, params, callback) {
      var interceptor = requestInterceptors[datasetId] || defaults.requestInterceptor;
      return interceptor(datasetId, params, callback);
    }

function ( datasetId , params , callback ) { var interceptor = requestInterceptors [ datasetId ] || defaults . requestInterceptor ; return interceptor ( datasetId , params , callback ) ; }

Can be used to check the request parameters or perform authentication checks
@param {String} datasetId
@param {Object} params
@param {Object} params.query_params
@param {Object} params.meta_data
@param {Function} callback


function addToQueue(items, extraParams, targetQueue, cb) {
  if (!items || items.length === 0) {
    return cb();
  }
  var itemsToPush = _.map(items, function(item) {
    return _.extend({}, item, extraParams);
  });
  debug("adding %d items to queue %s", itemsToPush.length, targetQueue.getName());
  targetQueue.addMany(itemsToPush, cb);
}

function
addToQueue
(
items
,
extraParams
,
targetQueue
,
cb
)
{
if
(
!
items
||
items
.
length
===
0
)
{
return
cb
(
)
;
}
var
itemsToPush
=
_
.
map
(
items
,
function
(
item
)
{
return
_
.
extend
(
{
}
,
item
,
extraParams
)
;
}
)
;
debug
(
"adding %d items to queue %s"
,
itemsToPush
.
length
,
targetQueue
.
getName
(
)
)
;
targetQueue
.
addMany
(
itemsToPush
,
cb
)
;
}

Add the given items to the given queue. For each item, also add the given extraParams.
@param {Array} items the items to add to the queue
@param {Object} extraParams extra data that should be added to each item
@param {MongodbQueue} targetQueue the queue to push the messages to
@param {Function} cb the callback function

Add the given items to the given queue . For each item also add the given extraParams .


function formatUpdates(processedUpdates) {
  var updates = {
    hashes: {}
  };
  _.each(processedUpdates, function(update) {
    var type = update.type;
    var hash = update.hash;
    updates.hashes[hash] = update;
    updates[type] = updates[type] || {};
    updates[type][hash] = update;
  });
  return updates;
}

function formatUpdates ( processedUpdates ) { var updates = { hashes : { } } ; _ . each ( processedUpdates , function ( update ) { var type = update . type ; var hash = update . hash ; updates . hashes [ hash ] = update ; updates [ type ] = updates [ type ] || { } ; updates [ type ] [ hash ] = update ; } ) ; return updates ; }

Reformat the give processedUpdates array to an object format that is expected by the sync client
@param {Array} processedUpdates the array of updates to process
@returns {Object} an object that contains the updates with different types


function removeUpdatesInRequest(updatesInDb, updatesInRequest) {
  var updatesNotInRequest = _.filter(updatesInDb, function(dbUpdate) {
    var foundInRequest = _.findWhere(updatesInRequest, {hash: dbUpdate.hash});
    return !foundInRequest;
  });
  return updatesNotInRequest;
}

function removeUpdatesInRequest ( updatesInDb , updatesInRequest ) { var updatesNotInRequest = _ . filter ( updatesInDb , function ( dbUpdate ) { var foundInRequest = _ . findWhere ( updatesInRequest , { hash : dbUpdate . hash } ) ; return ! foundInRequest ; } ) ; return updatesNotInRequest ; }

Remove all the records in `updatesInRequest` from `updatesInDb` if they exist
@param {Array} updatesInDb
@param {Array} updatesInRequest
@returns

Remove all the records in updatesInRequest from updatesInDb if they exist


function sync(datasetId, params, cb) {
  debug('[%s] process sync request for the dataset', datasetId);
  var queryParams = params.query_params || {};
  var metaData = params.meta_data || {};
  var datasetClient = new DatasetClient(datasetId, {queryParams: queryParams, metaData: metaData});
  debug('[%s] processing sync API request :: query_params = %j:: meta_data = %j', datasetId, queryParams, metaData);
  async.series({
    requestInterceptor: async.apply(interceptors.requestInterceptor, datasetId, params),
    readDatasetClient: function checkDatasetclientStopped(callback) {
      syncStorage.readDatasetClient(datasetClient.getId(), function(err, datasetClientJson) {
        if (err) {
          return callback(err);
        }
        if (datasetClientJson && datasetClientJson.stopped === true) {
          return callback(new Error('sync stopped for dataset ' + datasetId));
        } else {
          return callback(null, datasetClientJson);
        }
      });
    }
  }, function(err, results) {
    if (err) {
      debugError('[%s] sync request returns error = %s %j', datasetId, err, params);
      return cb(err);
    }
    return processSyncAPI(datasetId, params, results.readDatasetClient, cb);
  });
}

Process the sync request. It will
- validate the request body via the requestInterceptor
- check if the dataset client is stopped for sync
- create or update the dataset client
- process acknowledgements, push each of them to the ackQueue
- process pending changes, push each of them to the pendingQueue
- list any processed updates for the given client
@param {String} datasetId the id of the dataset
@param {Object} params the request body, it normally contain those fields:
@param {Object} params.query_params the query parameter for the dataset from the client
@param {Object} params.meta_data the meta data for the dataset from the client
@param {Object} params.__fh an object added by the client sdk. it could have the `cuid` of the client
@param {Array} params.pending the pending changes array
@param {Array} params.acknowledgements the acknowledgements from the client
@param {Function} cb the callback function

Process the sync request . It will - validate the request body via the requestInterceptor - check if the dataset client is stopped for sync - create or update the dataset client - process acknowledgements push each of them to the ackQueue - process pending changes push each of them to the pendingQueue - list any processed updates for the given client


function DatasetClientsCleaner(opts) {
  this.retentionPeriod = opts.retentionPeriod || '24h';
  this.checkFrequency = opts.checkFrequency || '1h';
  this.cleanerLockName = opts.cleanerLockName || 'locks:sync:DatasetCleaner';
  this.lockTimeout = parseDuration(this.checkFrequency) / 2;
  this.job;
}

function DatasetClientsCleaner ( opts ) { this . retentionPeriod = opts . retentionPeriod || '24h' ; this . checkFrequency = opts . checkFrequency || '1h' ; this . cleanerLockName = opts . cleanerLockName || 'locks:sync:DatasetCleaner' ; this . lockTimeout = parseDuration ( this . checkFrequency ) / 2 ; this . job ; }

Constructor for DatasetClientsCleaner.

A DatasetClientsCleaner will check the status of all the dataset clients in the db, and if any of them hasn't been accessed for the past <retentionPeriod> time, it will remove them from the db to save space.
By default, it will remove dataset clients that haven't been active for 1 day and it will run the check every hour.
Developer can change those parameters via the configuration.

@param {Object} opts configuration options for the DatasetClientsCleaner.
@param {String} retentionPeriod the maximum time the dataset client can be kept in db after it's last access tim. Support format: https://www.npmjs.com/package/parse-duration. Default is '24h'.
@param {String} checkFrequency how often the checks should be running. Default is '1h'.


function (error) {
                if (error) {
                    result.callbackError(error, false);
                } else {
                    result.callbackOk(undefined, false);
                }
            }

function ( error ) { if ( error ) { result . callbackError ( error , false ) ; } else { result . callbackOk ( undefined , false ) ; } }

if request contains invalid args, the invocation framework will provide error in callback no validation done here

if request contains invalid args the invocation framework will provide error in callback no validation done here


function (element) {
    for (var i = 0, len = this.length; i < len; i++) {
      if (element === this[i]) {
        return true;
      }
    }
    return false;
  }

function ( element ) { for ( var i = 0 , len = this . length ; i < len ; i ++ ) { if ( element === this [ i ] ) { return true ; } } return false ; }

Check if an element is in the list

@param element
@returns {boolean}


function (elements) {
    if (!_.isArray(elements)) {
      elements = [ elements ];
    }
    elements.forEach(function (e) {
      this.push(e);
    }.bind(this));
    return this;
  }

function ( elements ) { if ( ! _ . isArray ( elements ) ) { elements = [ elements ] ; } elements . forEach ( function ( e ) { this . push ( e ) ; } . bind ( this ) ) ; return this ; }

Add elements to the list

@param elements
@returns {ArrayList}


function (list) {
    var result = false;
    list.forEach(function (e) {
      result |= this.removeElement(e);
    }.bind(this));
    return result;
  }

function ( list ) { var result = false ; list . forEach ( function ( e ) { result |= this . removeElement ( e ) ; } . bind ( this ) ) ; return result ; }

Remove all the elements in another collection

@param list
@returns {boolean}


function (predicate) {
    var lists = new ArrayList;
    var arrays = _.partition(this, predicate);
    arrays.forEach(function (arr) {
      var list = new ArrayList;
      lists.push(list.add(arr));
    });
    return lists;
  }

function ( predicate ) { var lists = new ArrayList ; var arrays = _ . partition ( this , predicate ) ; arrays . forEach ( function ( arr ) { var list = new ArrayList ; lists . push ( list . add ( arr ) ) ; } ) ; return lists ; }

Split the list into two lists

@param predicate
@returns {ArrayList}


function (isSorted, iterator) {
    var list = new ArrayList;
    return list.add(_.uniq(this, isSorted, iterator));
  }

function ( isSorted , iterator ) { var list = new ArrayList ; return list . add ( _ . uniq ( this , isSorted , iterator ) ) ; }

Returns a list with the duplicated values removed

@param {boolean} isSorted
@param iterator
@returns {ArrayList}


function (iterator, context) {
    var list = new ArrayList;
    return list.add(_.map(this, iterator, context));
  }

function ( iterator , context ) { var list = new ArrayList ; return list . add ( _ . map ( this , iterator , context ) ) ; }

Returns a new list with each value mapped through a transformation

@param iterator
@param context
@returns {ArrayList}


function (predicate, context) {
    var list = new ArrayList;
    return list.add(_.filter(this, predicate, context));
  }

function ( predicate , context ) { var list = new ArrayList ; return list . add ( _ . filter ( this , predicate , context ) ) ; }

Returns a new list with the occurrences that passes the test

@param predicate
@param context
@returns {ArrayList}


function (iterator, context) {
    var list = new ArrayList;
    return list.add(_.sortBy(this, iterator, context));
  }

function ( iterator , context ) { var list = new ArrayList ; return list . add ( _ . sortBy ( this , iterator , context ) ) ; }

Returns a new list with the values sorted

@param iterator
@param context
@returns {ArrayList}


function thisToArgs(that, args) {
  args = Array.prototype.slice.call(args, 0);
  args.unshift(that);
  return args;
}

function thisToArgs ( that , args ) { args = Array . prototype . slice . call ( args , 0 ) ; args . unshift ( that ) ; return args ; }

Helper to use Function.prototype.call and prepend this to arguments

@param that
@param args
@returns {Array}

Helper to use Function . prototype . call and prepend this to arguments


function KoaNunjucks(oCtx, sPath, oOpts) {

    // Configure Nunjucks
    this.enviornment = nunjucks.configure(sPath, oOpts);

    // Save context
    this._ctx = oCtx;
}

function
KoaNunjucks
(
oCtx
,
sPath
,
oOpts
)
{
// Configure Nunjucks
this
.
enviornment
=
nunjucks
.
configure
(
sPath
,
oOpts
)
;
// Save context
this
.
_ctx
=
oCtx
;
}

Koa Nunjucks Class

@param {object} oCtx - Koa Context
@param {string} [sPath] - Path string
@param {object} [oOpts] - Options
@constructor


function get_registers(cpu) {
  cpu = cpu.toLowerCase();

  if (cpu.startsWith('arm')) {
    return REGISTERS.ARM;
  }

  if (~[ 'x86', 'i386', 'i486', 'i686' ].indexOf(cpu)) {
    return REGISTERS.X86;
  }

  if (cpu === 'x86_64') {
    return REGISTERS.X86_64;
  }

  return [];
}

function get_registers ( cpu ) { cpu = cpu . toLowerCase ( ) ; if ( cpu . startsWith ( 'arm' ) ) { return REGISTERS . ARM ; } if ( ~ [ 'x86' , 'i386' , 'i486' , 'i686' ] . indexOf ( cpu ) ) { return REGISTERS . X86 ; } if ( cpu === 'x86_64' ) { return REGISTERS . X86_64 ; } return [ ] ; }

Returns an array of registers associated with
a given CPU arch.

@param cpu the CPU arch.
@returns {Array} an array of registers.

Returns an array of registers associated with a given CPU arch .


function parse(report) {
  var sys  = report['system'] || {};
  var info = report['report'] || {};

  function _i(x) {
    return info[x] || '';
  }

  function _s(x) {
    return sys[x] || '';
  }

  var error  = report['crash']['error'];
  var thread = Utils.get_crash_thread(report);

  return [
    Utils.header('Incident Identifier:', _i('id'), 4),
    Utils.header('CrashReporter Key:'  , _s('device_app_hash'), 4),
    Utils.header('Hardware Model:'     , _s('machine'), 4),
    Utils.header('Process:'            , `${_s('process_name')} [${_s('process_id')}]`),
    Utils.header('Path:'               , _s('CFBundleExecutablePath')),
    Utils.header('Identifier:'         , _s('CFBundleIdentifier')),
    Utils.header('Version:'            , `${_s('CFBundleShortVersionString')} (${_s('CFBundleVersion')})`),
    Utils.header('Code Type:'          , get_cpu_arch(report)),
    Utils.header('Parent Process:'     , `${_s('parent_process_name')} [${_s('parent_process_id')}]`),
    Utils.header(''                    , ''),
    Utils.header('Date/Time:'          , get_time(report)),
    Utils.header('OS Version:'         , `${_s('system_name')} ${_s('system_version')} (${_s('os_version')})`),
    Utils.header('Report Version:'     , 104),
    Utils.header(''                    , '')
  ].concat(
    Errors.parse_errors(error, thread)
  );
}

function
parse
(
report
)
{
var
sys
=
report
[
'system'
]
||
{
}
;
var
info
=
report
[
'report'
]
||
{
}
;
function
_i
(
x
)
{
return
info
[
x
]
||
''
;
}
function
_s
(
x
)
{
return
sys
[
x
]
||
''
;
}
var
error
=
report
[
'crash'
]
[
'error'
]
;
var
thread
=
Utils
.
get_crash_thread
(
report
)
;
return
[
Utils
.
header
(
'Incident Identifier:'
,
_i
(
'id'
)
,
4
)
,
Utils
.
header
(
'CrashReporter Key:'
,
_s
(
'device_app_hash'
)
,
4
)
,
Utils
.
header
(
'Hardware Model:'
,
_s
(
'machine'
)
,
4
)
,
Utils
.
header
(
'Process:'
,
`
${
_s
(
'process_name'
)
}
${
_s
(
'process_id'
)
}
`
)
,
Utils
.
header
(
'Path:'
,
_s
(
'CFBundleExecutablePath'
)
)
,
Utils
.
header
(
'Identifier:'
,
_s
(
'CFBundleIdentifier'
)
)
,
Utils
.
header
(
'Version:'
,
`
${
_s
(
'CFBundleShortVersionString'
)
}
${
_s
(
'CFBundleVersion'
)
}
`
)
,
Utils
.
header
(
'Code Type:'
,
get_cpu_arch
(
report
)
)
,
Utils
.
header
(
'Parent Process:'
,
`
${
_s
(
'parent_process_name'
)
}
${
_s
(
'parent_process_id'
)
}
`
)
,
Utils
.
header
(
''
,
''
)
,
Utils
.
header
(
'Date/Time:'
,
get_time
(
report
)
)
,
Utils
.
header
(
'OS Version:'
,
`
${
_s
(
'system_name'
)
}
${
_s
(
'system_version'
)
}
${
_s
(
'os_version'
)
}
`
)
,
Utils
.
header
(
'Report Version:'
,
104
)
,
Utils
.
header
(
''
,
''
)
]
.
concat
(
Errors
.
parse_errors
(
error
,
thread
)
)
;
}

Parses out the Utils.headers of a crash report. The  Utils.headers
are the bulk of the meta information at the top of a
crash report.

@param report the JSON crash report.
@returns {string[]} an array of Utils.headers.

Parses out the Utils . headers of a crash report . The Utils . headers are the bulk of the meta information at the top of a crash report .


function convert(report) {
  return []
    .concat(
      Parsers.parse('headers', report),
      Parsers.parse('reason',  report),
      Parsers.parse('threads', report),
      Parsers.parse('cpu',     report),
      Parsers.parse('images',  report),
      Parsers.parse('extras',  report)
    )
    .join('\n')
}

function convert ( report ) { return [ ] . concat ( Parsers . parse ( 'headers' , report ) , Parsers . parse ( 'reason' , report ) , Parsers . parse ( 'threads' , report ) , Parsers . parse ( 'cpu' , report ) , Parsers . parse ( 'images' , report ) , Parsers . parse ( 'extras' , report ) ) . join ( '\n' ) }

Converts an input JSON report to an Apple style crash
report using the internal parses in sequence.

@param report the JSON report.
@returns {string} an Apple style report.

Converts an input JSON report to an Apple style crash report using the internal parses in sequence .


function get_crash_thread(report) {
  var crash   = report['crash'] || {};
  var threads = crash['threads'] || [];

  var thread = threads.find(function (thread) {
    return !!thread['crashed'];
  });

  return thread || crash['crashed_thread'];
}

function get_crash_thread ( report ) { var crash = report [ 'crash' ] || { } ; var threads = crash [ 'threads' ] || [ ] ; var thread = threads . find ( function ( thread ) { return ! ! thread [ 'crashed' ] ; } ) ; return thread || crash [ 'crashed_thread' ] ; }

Finds a crashed thread by iterating all threads and checking
whether they've crashed. If they have we short circuit and
return the thread. Otherwise we return the crashed_thread entry.

@param report the JSON report structure.
@returns {*} the entry of the crashed thread.

Finds a crashed thread by iterating all threads and checking whether they ve crashed . If they have we short circuit and return the thread . Otherwise we return the crashed_thread entry .


function parse(report, thread) {
  var rows = [''];
  var crashed = thread || Utils.get_crash_thread(report);

  if (!crashed) {
    return rows;
  }

  var index = crashed['index'];
  var sys   = report['system'] || {};
  var type  = sys['binary_cpu_type'];
  var sub   = sys['binary_cpu_subtype'];

  var arch;
  if (!type && !sub) {
    arch = sys['cpu_arch'];
  } else {
    arch = CPU.get_cpu_arch(type, sub);
  }

  var cpu = CPU.get_cpu_type(arch);

  rows.push(`Thread ${index} crashed with ${cpu} Thread State:`);

  var registers = (crashed['registers'] || {})['basic'] || {};
  var reg_order = CPU.get_registers(cpu);

  var line = '';

  for (var i = 0, j = reg_order.length; i < j; i++) {
    if (i % 4 === 0 && i !== 0) {
      rows.push(line);
      line = '';
    }

    var register      = reg_order[i];
    var register_addr = registers[register] || 0;
    var register_name = Utils.pad_left(register, ' ', 6);
    var register_loc  = Utils.pad_hex(register_addr, '0', 8);
    var register_pad  = Utils.pad_right(register_loc, ' ', 9);

    line += `${register_name}: 0x${register_pad}`;
  }

  if (line) {
    rows.push(line);
  }

  return rows;
}

function
parse
(
report
,
thread
)
{
var
rows
=
[
''
]
;
var
crashed
=
thread
||
Utils
.
get_crash_thread
(
report
)
;
if
(
!
crashed
)
{
return
rows
;
}
var
index
=
crashed
[
'index'
]
;
var
sys
=
report
[
'system'
]
||
{
}
;
var
type
=
sys
[
'binary_cpu_type'
]
;
var
sub
=
sys
[
'binary_cpu_subtype'
]
;
var
arch
;
if
(
!
type
&&
!
sub
)
{
arch
=
sys
[
'cpu_arch'
]
;
}
else
{
arch
=
CPU
.
get_cpu_arch
(
type
,
sub
)
;
}
var
cpu
=
CPU
.
get_cpu_type
(
arch
)
;
rows
.
push
(
`
${
index
}
${
cpu
}
`
)
;
var
registers
=
(
crashed
[
'registers'
]
||
{
}
)
[
'basic'
]
||
{
}
;
var
reg_order
=
CPU
.
get_registers
(
cpu
)
;
var
line
=
''
;
for
(
var
i
=
0
,
j
=
reg_order
.
length
;
i
<
j
;
i
++
)
{
if
(
i
%
4
===
0
&&
i
!==
0
)
{
rows
.
push
(
line
)
;
line
=
''
;
}
var
register
=
reg_order
[
i
]
;
var
register_addr
=
registers
[
register
]
||
0
;
var
register_name
=
Utils
.
pad_left
(
register
,
' '
,
6
)
;
var
register_loc
=
Utils
.
pad_hex
(
register_addr
,
'0'
,
8
)
;
var
register_pad
=
Utils
.
pad_right
(
register_loc
,
' '
,
9
)
;
line
+=
`
${
register_name
}
${
register_pad
}
`
;
}
if
(
line
)
{
rows
.
push
(
line
)
;
}
return
rows
;
}

Parses out the CPU registers dump from the crash. This is
reused internally and so you can pass a custom thread to
work against as needed.

@param report the JSON crash report.
@param [thread] an optional thread object.
@returns {string[]} an array of output.

Parses out the CPU registers dump from the crash . This is reused internally and so you can pass a custom thread to work against as needed .


function parse(report) {
  var system = report['system'] || {};
  var crash  = report['crash'] || {};

  return [ '', 'Extra Information:' ].concat(
    parse_nsexception(crash),
    parse_crash_thread(report),
    parse_last_exception(report),
    parse_diagnosis(crash),
    parse_recrash(report)
  );
}

function
parse
(
report
)
{
var
system
=
report
[
'system'
]
||
{
}
;
var
crash
=
report
[
'crash'
]
||
{
}
;
return
[
''
,
'Extra Information:'
]
.
concat
(
parse_nsexception
(
crash
)
,
parse_crash_thread
(
report
)
,
parse_last_exception
(
report
)
,
parse_diagnosis
(
crash
)
,
parse_recrash
(
report
)
)
;
}

Parses out the extra information available against the
crash. this includes any referenced objects, application
statistics and diagnoses.

@param report the JSON crash report.
@returns {string[]} an array of output.

Parses out the extra information available against the crash . this includes any referenced objects application statistics and diagnoses .


function parse_errors(error, thread) {
  var signal = error['signal'];
  var mach   = error['mach'];

  var exc_name  = mach['exception_name'] || '0';
  var code_name = mach['code_name'] || '0x00000000';
  var sig_name  = signal['name'] || signal['signal'] || '0';
  var addr_name = Utils.pad_hex(error['address'] || 0, '0', 8);

  var index  = 0;

  if (thread) {
    index = thread['index'];
  }

  return [
    Utils.header('Exception Type:'     , `${exc_name} (${sig_name})`),
    Utils.header('Exception Codes:'    , `${code_name} at 0x${addr_name}`),
    Utils.header('Crashed Thread:'     , index)
  ];
}

function
parse_errors
(
error
,
thread
)
{
var
signal
=
error
[
'signal'
]
;
var
mach
=
error
[
'mach'
]
;
var
exc_name
=
mach
[
'exception_name'
]
||
'0'
;
var
code_name
=
mach
[
'code_name'
]
||
'0x00000000'
;
var
sig_name
=
signal
[
'name'
]
||
signal
[
'signal'
]
||
'0'
;
var
addr_name
=
Utils
.
pad_hex
(
error
[
'address'
]
||
0
,
'0'
,
8
)
;
var
index
=
0
;
if
(
thread
)
{
index
=
thread
[
'index'
]
;
}
return
[
Utils
.
header
(
'Exception Type:'
,
`
${
exc_name
}
${
sig_name
}
`
)
,
Utils
.
header
(
'Exception Codes:'
,
`
${
code_name
}
${
addr_name
}
`
)
,
Utils
.
header
(
'Crashed Thread:'
,
index
)
]
;
}

Parses out the exception section of a Crash report.

@param error the error object to deal with.
@param thread the crashed thread.
@returns {*[]} an array of output.

Parses out the exception section of a Crash report .


function(definedLogLevels) {
		logLevels = this._merge(definedLogLevels);

		// go through all the loggers
		for (var key in winston.loggers.loggers)  {
			var logger = winston.loggers.loggers[key];

			if (logLevels['all']) {
				if (logLevels['all'] != 'none') { // turning on everything
					if (logger.transports.console) {
						logger.transports.console.level = logLevels['all'];
					}
					else {
						logger.add(winston.transports.Console, {
							level		: logLevels['all'],
							json 		: false,
					    	timestamp 	: true,
					    	label		: key,
					    	align		: true
						}, false);
					}
				}
				else { // all = none so remove everything
					logger.remove(winston.transports.Console)
				}
			}
			else {
				// individual log levels were set
				var level = logLevels[key];
				if (logger.transports.console) { // if there isn't even a console, nothing to do
					if (level != 'none') {
						logger.transports.console.level = level;
					}
					else { // level = none, so turn it off
						logger.remove(winston.transports.Console)
					}
				}
			}
		}
		return this;
	}

Allows for runtime setting of log levels.  Can be called at anytime to adjust log levels.

The passing in map specifies the logging level of the object to have logging enabled.
@example
{
"MyObject" : "debug",
"SomeOtherObject" : "info",
"YAObject" : "none"
}

You can also pass in a object name (key) of "all" to apply a certain level to all your objects.
@example
{
"all" : "debug",
"SomeOtherObject" : "info"
}
This will set every object to "debug" except SomeOtherObject, which will be set to "info".

@function
@param {Map} definedLogLevels The logging levels for any object

Allows for runtime setting of log levels . Can be called at anytime to adjust log levels .


function parse(report) {
  var rows = [''];

  var crash  = report['crash'] || {};
  var error  = crash['error'];
  var type   = error['type'];
  var reason = error['reason'];

  var user_exception = error['user_reported'];
  var ns_exception   = error['nsexception'];

  if (ns_exception) {
    rows.push(format(ns_exception));
    rows.push('');
    return rows;
  }

  if (zombie_exception(report)) {
    var last_exception = Utils
      .get_last_exception(report);

    if (last_exception) {
      rows.push(format(last_exception, reason));
      rows.push('NOTE: This exception has been deallocated! ' +
        'Stack trace is crash from attempting to access this zombie exception.');
      rows.push('');
    }

    return rows;
  }

  if (user_exception) {
    rows.push(format(user_exception, reason));

    var line = user_exception['line_of_code'];
    var back = user_exception['backtrace'] || [];

    if (line || back) {
      rows.push('Custom Backtrace:');
    }

    if (line) {
      rows.push(`Line: ${line}`);
    }

    return rows.concat(back);
  }

  if (type == 'cpp_exception') {
    return rows.concat(format(error['cppexception'], reason));
  }

  if (type == 'deadlock') {
    rows.push('Application main thread deadlocked');
    rows.push('');
  }

  return rows;
}

Parses out the reason for why the error occurred. We take
this from the error sub-object, unless a user exception is
provided and if so we use that.

@param report the JSON crash report.
@returns {string[]} an array of output.

Parses out the reason for why the error occurred . We take this from the error sub - object unless a user exception is provided and if so we use that .


function order_stack (stack) {
    // Find a handle with a before or after property
    for (var i = 0; i < stack.length; i++) {
      var handle = stack[i].handle;
      if (handle._first) {
        // remove handle from current position
        var mid = stack.splice(i, 1)[0];
        // insert it at begining of stack
        stack.unshift(mid);

        // remove property so we don't order it again later
        delete handle._first;
        // for debugging
        handle._moved_first = true;

        // Continue ordering for remaining handles
        return order_stack (stack);
      }
      else if (handle._before || handle._after) {
        var position = null;
        if (handle._before) {
          position = '_before';
        }
        else if (handle._after) {
          position = '_after';
        }

        var label = handle[position];

        for (var j = 0; j < stack.length; j++) {
          if (stack[j].handle.label === label) {
            // insert before index = j
            // insert after index = j + 1
            var new_index = j;
            if (position == '_after') new_index++;

            // move handle in new position
            // http://stackoverflow.com/questions/5306680/move-an-array-element-from-one-array-position-to-another
            stack.splice(new_index, 0, stack.splice(i, 1)[0]);

            // remove _before/_after property so we don't order ad infinitum
            handle['_moved' + position] = handle[position]; // for debugging
            break;
          }
        }
        delete handle[position];
        // Continue ordering for remaining handles
        return order_stack (stack);
      }
    }
    // didn't find any handle with a before/after property => done ordering
    return true;
  }

lookup connect.stack if there is a fn with before or after properties and move it at right place @todo: optimize

lookup connect . stack if there is a fn with before or after properties and move it at right place


function(name, src, onload, requester, errorback) {

        // XHR because it's easier to work with, doesn't pollute and has no reliance on DOM.
        // This means support for web workers, possibility to add a pre-processing pipeline
        // along with potential to cache scripts for re-use in other contexts.
        var xhr = new XMLHttpRequest();
        xhr.open("GET", src, true);
        var onerror = function(e) {
            if (errorback) {
                errorback({
                    src: src,
                    requester: requester
                });
            } else {
                throw new Error('\n  Missing: ' + src + '\n  Requester: ' + requester);
            }
        };
        xhr.onerror = onerror;
        xhr.onload = function() {
            if (xhr.status >= 200 && xhr.status < 300 || xhr.status === 304) {
                var output = xhr.responseText;
                output += "\n//# sourceURL=" + self.location.protocol + "//" + resolveRelative(self.location.host + self.location.pathname, src);
                eval.call(null, output);
                onload();
            } else {
                onerror();
            }
        };
        xhr.send();
    }

Transport is separated into a separate function because it can be overrided by the lamda-optimizer tool to use NodeJS functions instead.

Transport is separated into a separate function because it can be overrided by the lamda - optimizer tool to use NodeJS functions instead .


function rgbToHex(rgb) {

    function componentToHex(c) {
      var hex = c.toString(16).toUpperCase();
      return hex.length === 1 ? "0" + hex : hex;
    }

    var color = rgb.map(function(component) {
      return componentToHex(parseInt(component * 255));
    });

    color.unshift("0X");
    color = color.join("");
    return parseInt(color);

  }

function rgbToHex ( rgb ) { function componentToHex ( c ) { var hex = c . toString ( 16 ) . toUpperCase ( ) ; return hex . length === 1 ? "0" + hex : hex ; } var color = rgb . map ( function ( component ) { return componentToHex ( parseInt ( component * 255 ) ) ; } ) ; color . unshift ( "0X" ) ; color = color . join ( "" ) ; return parseInt ( color ) ; }

Converts decimal values in the range 0-1 to hexadecimal values between 00 and FF.
@param {Array} rgb: An array of three values, representing red, green and blue values.

Converts decimal values in the range 0 - 1 to hexadecimal values between 00 and FF .


function addKnockoutArrayMutators(ko, array, subscribable, signal) {
  var fnNames = ['remove', 'removeAll', 'destroy', 'destroyAll', 'replace'];
  fnNames.forEach(function(fnName) {
    // Make it a non-enumerable property for consistency with standard Array
    // functions
    Object.defineProperty(array, fnName, {
      enumerable: false,
      value: function() {
        var result;

        // These additional array mutators are built using the underlying
        // push/pop/etc. mutators, which are wrapped to trigger notifications.
        // But we don't want to trigger multiple notifications, so pause the
        // push/pop/etc. wrappers and delivery only one notification at the end
        // of the process.
        signal.pause = true;
        try {
          // Creates a temporary observableArray that can perform the operation.
          var fn = ko.observableArray.fn[fnName];
          result = fn.apply(ko.observableArray(array), arguments);
        }
        finally {
          signal.pause = false;
        }

        subscribable.notifySubscribers(array);
        return result;
      }
    });
  });
}

Adds Knockout's additional array mutation functions to the array

Adds Knockout s additional array mutation functions to the array


function getSubscribableForArray(ko, array) {
  var subscribable = array._subscribable;
  var signal = {};

  if (!subscribable) {
    subscribable = array._subscribable = new ko.subscribable();

    wrapStandardArrayMutators(array, subscribable, signal);
    addKnockoutArrayMutators(ko, array, subscribable, signal);
  }

  return subscribable;
}

function getSubscribableForArray ( ko , array ) { var subscribable = array . _subscribable ; var signal = { } ; if ( ! subscribable ) { subscribable = array . _subscribable = new ko . subscribable ( ) ; wrapStandardArrayMutators ( array , subscribable , signal ) ; addKnockoutArrayMutators ( ko , array , subscribable , signal ) ; } return subscribable ; }

Gets or creates a subscribable that fires after each array mutation


function startWatchingarray(ko, observable, array) {
  var subscribable = getSubscribableForArray(ko, array);
  return subscribable.subscribe(observable);
}

function startWatchingarray ( ko , observable , array ) { var subscribable = getSubscribableForArray ( ko , array ) ; return subscribable . subscribe ( observable ) ; }

Listens for array mutations, and when they happen, cause the observable to fire notifications. This is used to make model properties of type array fire notifications when the array changes. Returns a subscribable that can later be disposed.

Listens for array mutations and when they happen cause the observable to fire notifications . This is used to make model properties of type array fire notifications when the array changes . Returns a subscribable that can later be disposed .


function notifyWhenPresentOrFutureArrayValuesMutate(ko, observable) {
  var watchingArraySubscription = null;
  ko.computed(function() {
    // Unsubscribe to any earlier array instance
    if (watchingArraySubscription) {
      watchingArraySubscription.dispose();
      watchingArraySubscription = null;
    }

    // Subscribe to the new array instance
    var newarray = observable();
    if (newarray instanceof Array) {
      watchingArraySubscription = startWatchingarray(ko, observable, newarray);
    }
  });
}

Given an observable that underlies a model property, watch for any array value that might be assigned as the property value, and hook into its change events

Given an observable that underlies a model property watch for any array value that might be assigned as the property value and hook into its change events


function deleteBackup(config) {

	return new Promise((resolve, reject) => {

		const backupPath = path.join(process.cwd(), (config.backup !== undefined) ? config.backup : ".backup");

		fs.remove(backupPath, (error) => {

			(error === null) ? resolve(config) : reject(error);

		});

	});

}

function deleteBackup ( config ) { return new Promise ( ( resolve , reject ) => { const backupPath = path . join ( process . cwd ( ) , ( config . backup !== undefined ) ? config . backup : ".backup" ) ; fs . remove ( backupPath , ( error ) => { ( error === null ) ? resolve ( config ) : reject ( error ) ; } ) ; } ) ; }

Deletes the backup directory.

@param {Object} config - A configuration.
@return {Promise} A promise that returns the given configuration.


function inline(config, files) {

	return new Promise((resolve, reject) => {

		let filesChanged = 0;
		let i = 0;

		(function proceed(result) {

			if(result) {

				filesChanged++;

			}

			if(i === files.length) {

				resolve("Modified " + filesChanged + " files");

			} else {

				InlineImport.transform(files[i++], config.options).then(proceed).catch(reject);

			}

		}());

	});

}

function
inline
(
config
,
files
)
{
return
new
Promise
(
(
resolve
,
reject
)
=>
{
let
filesChanged
=
0
;
let
i
=
0
;
(
function
proceed
(
result
)
{
if
(
result
)
{
filesChanged
++
;
}
if
(
i
===
files
.
length
)
{
resolve
(
"Modified "
+
filesChanged
+
" files"
)
;
}
else
{
InlineImport
.
transform
(
files
[
i
++
]
,
config
.
options
)
.
then
(
proceed
)
.
catch
(
reject
)
;
}
}
(
)
)
;
}
)
;
}

Inlines the given files.

@param {Object} config - A configuration.
@param {String[]} files - The files.
@return {Promise} A promise that returns an info message.


function getFiles(config) {

	return new Promise((resolve, reject) => {

		const src = config.src;
		let files = [];
		let i = 0;

		(function proceed(error, moreFiles) {

			if(error !== null) {

				reject(error);

			} else {

				files = files.concat(moreFiles);

				if(i === src.length) {

					if(files.length === 0) {

						reject("No input files found");

					} else {

						resolve([config, files]);

					}

				} else {

					glob(src[i++], proceed);

				}

			}

		}(null, []));

	});

}

function
getFiles
(
config
)
{
return
new
Promise
(
(
resolve
,
reject
)
=>
{
const
src
=
config
.
src
;
let
files
=
[
]
;
let
i
=
0
;
(
function
proceed
(
error
,
moreFiles
)
{
if
(
error
!==
)
{
reject
(
error
)
;
}
else
{
files
=
files
.
concat
(
moreFiles
)
;
if
(
i
===
src
.
length
)
{
if
(
files
.
length
===
0
)
{
reject
(
"No input files found"
)
;
}
else
{
resolve
(
[
config
,
files
]
)
;
}
}
else
{
glob
(
src
[
i
++
]
,
proceed
)
;
}
}
}
(
,
[
]
)
)
;
}
)
;
}

Gathers files that will be modified.

@type {Object} config - A configuration.
@return {Promise} A promise that returns an array containing the given configuration and the identified files.


function validateConfig(config) {

	return new Promise((resolve, reject) => {

		if(config.src !== undefined) {

			if(!Array.isArray(config.src)) {

				config.src = [config.src];

			}

			resolve(config);

		} else {

			reject("No source path specified");

		}

	});

}

function
validateConfig
(
config
)
{
return
new
Promise
(
(
resolve
,
reject
)
=>
{
if
(
config
.
src
!==
undefined
)
{
if
(
!
Array
.
isArray
(
config
.
src
)
)
{
config
.
src
=
[
config
.
src
]
;
}
resolve
(
config
)
;
}
else
{
reject
(
"No source path specified"
)
;
}
}
)
;
}

Validates a given configuration.

@param {Object} config - A configuration.
@return {Promise} A promise that returns the given configuration.


function readConfig() {

	// Checks if the package file contains a configuration.
	const pkgConfigPromise = new Promise((resolve, reject) => {

		fs.readFile(path.join(process.cwd(), "package.json")).then((data) => {

			try {

				resolve(JSON.parse(data).inlineImport);

			} catch(error) {

				reject(error);

			}

		}).catch((error) => {

			// There's no configuration in package.json.
			resolve();

		});

	});

	// Looks for a configuration file.
	const configFilePromise = new Promise((resolve, reject) => {

		fs.readFile(path.join(process.cwd(), argv.config)).then((data) => {

			try {

				resolve(JSON.parse(data));

			} catch(error) {

				reject(error);

			}

		}).catch((error) => {

			if(error.code === "ENOENT") {

				reject("No configuration found (" + argv.config + ")");

			} else {

				reject("Failed to read the configuration file (" + error + ")");

			}

		});

	});

	return new Promise((resolve, reject) => {

		pkgConfigPromise.then((config) => {

			return (config !== undefined) ? Promise.resolve(config) : configFilePromise;

		}).then(resolve).catch(reject);

	});

}

Loads the configuration.

@return {Promise} A promise that returns the configuration.


function readFile(file, encoding) {

	return new Promise((resolve, reject) => {

		fs.readFile(file, encoding, (error, data) => error ? reject(error) : resolve(data));

	});

}

function readFile ( file , encoding ) { return new Promise ( ( resolve , reject ) => { fs . readFile ( file , encoding , ( error , data ) => error ? reject ( error ) : resolve ( data ) ) ; } ) ; }

Reads the given file.

@private
@param {String} file - A file path.
@param {String} encoding - The file encoding.
@return {Promise} A promise.


function parseImports(data, file, extensions) {

	const imports = [];

	let result = importRegExp.exec(data);
	let encoding;

	while(result !== null) {

		encoding = extensions[path.extname(result[2])];

		// Filter irrelevant imports.
		if(encoding !== undefined) {

			imports.push(new FileImport(
				result.index,
				importRegExp.lastIndex,
				result[1],
				path.resolve(path.dirname(file), result[2]),
				encoding
			));

		}

		result = importRegExp.exec(data);

	}

	return Promise.resolve([imports, data]);

}

function
parseImports
(
data
,
file
,
extensions
)
{
const
imports
=
[
]
;
let
result
=
importRegExp
.
exec
(
data
)
;
let
encoding
;
while
(
result
!==
)
{
encoding
=
extensions
[
path
.
extname
(
result
[
2
]
)
]
;
// Filter irrelevant imports.
if
(
encoding
!==
undefined
)
{
imports
.
push
(
new
FileImport
(
result
.
index
,
importRegExp
.
lastIndex
,
result
[
1
]
,
path
.
resolve
(
path
.
dirname
(
file
)
,
result
[
2
]
)
,
encoding
)
)
;
}
result
=
importRegExp
.
exec
(
data
)
;
}
return
Promise
.
resolve
(
[
imports
,
data
]
)
;
}

Parses the file for import statements.

@private
@param {String} data - The file contents.
@param {String} file - The path of the source file.
@param {Object} extensions - Extensions of imports that shall be inlined.
@return {Promise} A promise.


function readImports(imports, data) {

	return (imports.length === 0) ? Promise.resolve([imports, data]) : new Promise((resolve, reject) => {

		let i = 0;

		(function proceed(error, importData) {

			if(importData) {

				imports[i++].data = importData;

			}

			if(error) {

				reject(error);

			} else if(i === imports.length) {

				resolve([imports, data]);

			} else {

				fs.readFile(imports[i].path, imports[i].encoding, proceed);

			}

		}());

	});

}

function readImports ( imports , data ) { return ( imports . length === 0 ) ? Promise . resolve ( [ imports , data ] ) : new Promise ( ( resolve , reject ) => { let i = 0 ; ( function proceed ( error , importData ) { if ( importData ) { imports [ i ++ ] . data = importData ; } if ( error ) { reject ( error ) ; } else if ( i === imports . length ) { resolve ( [ imports , data ] ) ; } else { fs . readFile ( imports [ i ] . path , imports [ i ] . encoding , proceed ) ; } } ( ) ) ; } ) ; }

Reads the contents of the imported files.

@private
@param {FileImport[]} imports - A list of relevant import statements.
@param {String} data - The file contents.
@return {Promise} A promise.

Reads the contents of the imported files .


function inlineImports(imports, data, declaration) {

	let modified = imports.length > 0;
	let i, item;

	// Inline the imports in reverse order to keep the indices intact.
	for(i = imports.length - 1; i >= 0; --i) {

		item = imports[i];

		data = data.substring(0, item.start) +
			declaration + " " + item.name + " = " + JSON.stringify(item.data) +
			data.substring(item.end);

	}

	return Promise.resolve([modified, data]);

}

Replaces the affected import statements with the actual file contents.

@private
@param {FileImport[]} imports - A list of all relevant imports.
@param {String} data - The original file contents.
@param {String} declaration - The import variable declaration.
@return {Promise} A promise.

Replaces the affected import statements with the actual file contents .


function writeFile(modified, data, file) {

	return !modified ? Promise.resolve(false) : new Promise((resolve, reject) => {

		fs.writeFile(file, data, (error) => {

			error ? reject(error) : resolve(true);

		});

	});

}

function writeFile ( modified , data , file ) { return ! modified ? Promise . resolve ( false ) : new Promise ( ( resolve , reject ) => { fs . writeFile ( file , data , ( error ) => { error ? reject ( error ) : resolve ( true ) ; } ) ; } ) ; }

Applies the changes by overwriting the original file.

@private
@param {Boolean} modified - Indicates whether the file contents have been modified.
@param {String} data - The modified file contents.
@param {String} file - The file path.
@return {Promise} A promise that will return false if the file remained unchanged, or true if it was modified.

Applies the changes by overwriting the original file .


function (N, data, CART_OR_SPH, DIRECT_OR_PINV) {

    var Ndirs = data.length, Nsh = (N+1)*(N+1);
    var invY_N;
    var mag = [,];
    if (Nsh>Ndirs)  {
        console.log("The SHT degree is too high for the number of data points")
    }

    // Convert cartesian to spherical if needed
    if (CART_OR_SPH==0) data = convertCart2Sph(data);
    for (var  i=0; i<data.length; i++) {
        mag[i] = data[i][2];
    }
    // SH sampling matrix
    Y_N = computeRealSH(N, data);
    // Direct SHT
    if (DIRECT_OR_PINV==0) {
        invY_N = numeric.mul(1/Ndirs,Y_N);
    }
    else {
        invY_N = pinv_direct(numeric.transpose(Y_N));
    }
    // Perform SHT
    var coeffs = numeric.dotMV(invY_N, mag);
    return coeffs;
}

forwardSHT implements the forward SHT on data defined over the sphere


function (coeffs, aziElev) {

    var aziElevR = aziElev;
    var N = Math.sqrt(coeffs.length)-1;
    // SH sampling matrix
    var Y_N = computeRealSH(N, aziElev);
    // reconstruction
    var data = numeric.dotVM(coeffs, Y_N);
    // gather in data matrix
    for (var i=0; i<aziElev.length; i++) {
        aziElevR[i][2] = data[i];
    }
    return aziElevR;
}

function
(
coeffs
,
aziElev
)
{
var
aziElevR
=
aziElev
;
var
N
=
Math
.
sqrt
(
coeffs
.
length
)
1
;
// SH sampling matrix
var
Y_N
=
computeRealSH
(
N
,
aziElev
)
;
// reconstruction
var
data
=
numeric
.
dotVM
(
coeffs
,
Y_N
)
;
// gather in data matrix
for
(
var
i
=
0
;
i
<
aziElev
.
length
;
i
++
)
{
aziElevR
[
i
]
[
2
]
=
data
[
i
]
;
}
return
aziElevR
;
}

inverseSHT implements the inverse SHT from SH coefficients


function (xyz, OMIT_MAG) {

    var azi, elev, r;
    var aziElevR = new Array(xyz.length);

    for (var i=0; i<xyz.length; i++) {
        azi = Math.atan2( xyz[i][1], xyz[i][0] );
        elev = Math.atan2( xyz[i][2], Math.sqrt(xyz[i][0]*xyz[i][0] + xyz[i][1]*xyz[i][1]) );
        if (OMIT_MAG==1) {
            aziElevR[i] = [azi,elev];
        }
        else {
            r = Math.sqrt(xyz[i][0]*xyz[i][0] + xyz[i][1]*xyz[i][1] + xyz[i][2]*xyz[i][2]);
            aziElevR[i] = [azi,elev,r];
        }
    }
    return aziElevR;
}

function ( xyz , OMIT_MAG ) { var azi , elev , r ; var aziElevR = new Array ( xyz . length ) ; for ( var i = 0 ; i < xyz . length ; i ++ ) { azi = Math . atan2 ( xyz [ i ] [ 1 ] , xyz [ i ] [ 0 ] ) ; elev = Math . atan2 ( xyz [ i ] [ 2 ] , Math . sqrt ( xyz [ i ] [ 0 ] * xyz [ i ] [ 0 ] + xyz [ i ] [ 1 ] * xyz [ i ] [ 1 ] ) ) ; if ( OMIT_MAG == 1 ) { aziElevR [ i ] = [ azi , elev ] ; } else { r = Math . sqrt ( xyz [ i ] [ 0 ] * xyz [ i ] [ 0 ] + xyz [ i ] [ 1 ] * xyz [ i ] [ 1 ] + xyz [ i ] [ 2 ] * xyz [ i ] [ 2 ] ) ; aziElevR [ i ] = [ azi , elev , r ] ; } } return aziElevR ; }

convertCart2Sph converts arrays of cartesian vectors to spherical coordinates