includes/backup.js
function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
  var total = 0; // running total of documents downloaded so far
  var noRemainingBatches = false;

  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
  // log file and download them. Set noRemainingBatches to `true` for last batch.
  function downloadSingleBatchSet(done) {
    // Fetch the doc IDs for the batches in the current set to
    // download them.
    function batchSetComplete(err, data) {
      total = data.total;
      done();
    }
    function processRetrievedBatches(err, batches) {
      // process them in parallelised queue
      processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
    }

    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
      if (err) {
        ee.emit('error', err);
        // Stop processing changes file for fatal errors
        noRemainingBatches = true;
        done();
      } else {
        if (batchSetIds.length === 0) {
          noRemainingBatches = true;
          return done();
        }
        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
      }
    });
  }

  // Return true if all batches in log file have been downloaded
  function isFinished() { return noRemainingBatches; }

  function onComplete() {
    ee.emit('finished', { total: total });
  }

  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
}
function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
  var total = 0; // running total of documents downloaded so far
  var noRemainingBatches = false;

  // Generate a set of batches (up to batchesPerDownloadSession) to download from the
  // log file and download them. Set noRemainingBatches to `true` for last batch.
  function downloadSingleBatchSet(done) {
    // Fetch the doc IDs for the batches in the current set to
    // download them.
    function batchSetComplete(err, data) {
      total = data.total;
      done();
    }
    function processRetrievedBatches(err, batches) {
      // process them in parallelised queue
      processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
    }

    readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
      if (err) {
        ee.emit('error', err);
        // Stop processing changes file for fatal errors
        noRemainingBatches = true;
        done();
      } else {
        if (batchSetIds.length === 0) {
          noRemainingBatches = true;
          return done();
        }
        logfilegetbatches(log, batchSetIds, processRetrievedBatches);
      }
    });
  }

  // Return true if all batches in log file have been downloaded
  function isFinished() { return noRemainingBatches; }

  function onComplete() {
    ee.emit('finished', { total: total });
  }

  async.doUntil(downloadSingleBatchSet, isFinished, onComplete);
}
function downloadRemainingBatches ( log , db , ee , startTime , batchesPerDownloadSession , parallelism ) { var total = 0 ; // running total of documents downloaded so far var noRemainingBatches = false ; // Generate a set of batches (up to batchesPerDownloadSession) to download from the // log file and download them. Set noRemainingBatches to `true` for last batch. function downloadSingleBatchSet ( done ) { // Fetch the doc IDs for the batches in the current set to // download them. function batchSetComplete ( err , data ) { total = data . total ; done ( ) ; } function processRetrievedBatches ( err , batches ) { // process them in parallelised queue processBatchSet ( db , parallelism , log , batches , ee , startTime , total , batchSetComplete ) ; } readBatchSetIdsFromLogFile ( log , batchesPerDownloadSession , function ( err , batchSetIds ) { if ( err ) { ee . emit ( 'error' , err ) ; // Stop processing changes file for fatal errors noRemainingBatches = true ; done ( ) ; } else { if ( batchSetIds . length === 0 ) { noRemainingBatches = true ; return done ( ) ; } logfilegetbatches ( log , batchSetIds , processRetrievedBatches ) ; } } ) ; } // Return true if all batches in log file have been downloaded function isFinished ( ) { return noRemainingBatches ; } function onComplete ( ) { ee . emit ( 'finished' , { total : total } ) ; } async . doUntil ( downloadSingleBatchSet , isFinished , onComplete ) ; }
Download remaining batches in a log file, splitting batches into sets
to avoid enqueueing too many in one go.

@param {string} log - log file name to maintain download state
@param {string} db - nodejs-cloudant db
@param {events.EventEmitter} ee - event emitter to emit received events on
@param {time} startTime - start time for backup process
@param {number} batchesPerDownloadSession - max batches to enqueue for
download at a time. As batches contain many doc IDs, this helps avoid
exhausting memory.
@param {number} parallelism - number of concurrent downloads
@returns function to call do download remaining batches with signature
(err, {batches: batch, docs: doccount}) {@see spoolchanges}.
Download remaining batches in a log file splitting batches into sets to avoid enqueueing too many in one go .