function itemsCallback(error, itemsResponse) {
    if (error) throw error;

    var items = itemsResponse.searchResult.item;

    console.log('Found', items.length, 'items');

    for (var i = 0; i < items.length; i++) {
      console.log('- ' + items[i].title);
    }
  }

function
itemsCallback
(
error
,
itemsResponse
)
{
if
(
error
)
throw
error
;
var
items
=
itemsResponse
.
searchResult
.
item
;
console
.
log
(
'Found'
,
items
.
length
,
'items'
)
;
for
(
var
i
=
0
;
i
<
items
.
length
;
i
++
)
{
console
.
log
(
'- '
+
items
[
i
]
.
title
)
;
}
}

gets all the items together in a merged array


function getExe() {
  /* istanbul ignore next: tested on all platform on travis */
  switch (process.platform) {
    case "darwin":
      return "mkcert-" + MKCERT_VERSION + "-darwin-amd64"
    case "linux":
      return "mkcert-" + MKCERT_VERSION + "-linux-amd64"
    case "win32":
      return "mkcert-" + MKCERT_VERSION + "-windows-amd64.exe"
    default:
      console.warn("Cannot generate the localhost certificate on your " +
        "platform. Please, consider contacting the developer if you can help.")
      process.exit(0)
  }
}


function download(url, path) {
  console.log("Downloading the mkcert executable...")
  const file = fs.createWriteStream(path)
  return new Promise(resolve => {
    function get(url, file) {
      https.get(url, (response) => {
        if (response.statusCode === 302) get(response.headers.location, file)
        else response.pipe(file).on("finish", resolve)
      })
    }
    get(url, file)
  })
}

function
download
(
url
,
path
)
{
console
.
log
(
"Downloading the mkcert executable..."
)
const
file
=
fs
.
createWriteStream
(
path
)
return
new
Promise
(
resolve
=>
{
function
get
(
url
,
file
)
{
https
.
get
(
url
,
(
response
)
=>
{
if
(
response
.
statusCode
===
302
)
get
(
response
.
headers
.
location
,
file
)
else
response
.
pipe
(
file
)
.
on
(
"finish"
,
resolve
)
}
)
}
get
(
url
,
file
)
}
)
}


function mkcert(appDataPath, exe) {
  const logPath = path.join(appDataPath, "mkcert.log")
  const errPath = path.join(appDataPath, "mkcert.err")
  // escape spaces in appDataPath (Mac OS)
  appDataPath = appDataPath.replace(" ", "\\ ")
  const exePath = path.join(appDataPath, exe)
  const crtPath = path.join(appDataPath, "localhost.crt")
  const keyPath = path.join(appDataPath, "localhost.key")
  const cmd = exePath + " -install -cert-file " + crtPath +
    " -key-file " + keyPath + " localhost"
  return new Promise((resolve, reject) => {
    console.log("Running mkcert to generate certificates...")
    exec(cmd, (err, stdout, stderr) => {
      // log
      const errFun = err => {
        /* istanbul ignore if: cannot be tested */
        if (err) console.error(err)
      }
      fs.writeFile(logPath, stdout, errFun)
      fs.writeFile(errPath, stderr, errFun)
      /* istanbul ignore if: cannot be tested */
      if (err) reject(err)
      resolve()
    })
  })
}

execute the binary executable to generate the certificates


function remove(appDataPath = CERT_PATH) {
  if (fs.existsSync(appDataPath)) {
    fs.readdirSync(appDataPath)
      .forEach(file => fs.unlinkSync(path.join(appDataPath, file)))
    fs.rmdirSync(appDataPath)
  }
}

function remove ( appDataPath = CERT_PATH ) { if ( fs . existsSync ( appDataPath ) ) { fs . readdirSync ( appDataPath ) . forEach ( file => fs . unlinkSync ( path . join ( appDataPath , file ) ) ) fs . rmdirSync ( appDataPath ) } }


function clone(v) {
    if (v === null || typeof v !== "object") {
        return v;
    }

    if (isArray(v)) {
        var arr = v.slice();
        for (var i = 0; i < v.length; i++) {
            arr[i] = clone(arr[i]);
        }
        return arr;
    }
    else {
        var obj = {};
        for (var k in v) {
            obj[k] = clone(v[k]);
        }
        return obj;
    }
}

function clone ( v ) { if ( v === null || typeof v !== "object" ) { return v ; } if ( isArray ( v ) ) { var arr = v . slice ( ) ; for ( var i = 0 ; i < v . length ; i ++ ) { arr [ i ] = clone ( arr [ i ] ) ; } return arr ; } else { var obj = { } ; for ( var k in v ) { obj [ k ] = clone ( v [ k ] ) ; } return obj ; } }

Recursively copies given object into a new object. Helper method for merge

Recursively copies given object into a new object . Helper method for merge


function (identifier, data, config, cb) {
            if (data.tags === undefined) {
                return new Error("tags param can not be empty.");
            }
            var path = api.getPathWithData(baseURL, identifier, tagsActionPath, data);
            api.delete(path, config, cb);
        }

function
(
identifier
,
data
,
config
,
cb
)
{
if
(
data
.
tags
===
undefined
)
{
return
new
Error
(
"tags param can not be empty."
)
;
}
var
path
=
api
.
getPathWithData
(
baseURL
,
identifier
,
tagsActionPath
,
data
)
;
api
.
delete
(
path
,
config
,
cb
)
;
}

This request is to remove tags from an alert in Opsgenie.

One of id, tinyId or alias parameters should be specified with remove tags request as identifier.
Alias option can only be used for open alerts
example identifier objects:
{identifier : "123", identifierType : "tiny"}
{identifier : "123-23-123", identifierType : "id"}
{identifier : "alertAlias", identifierType : "alias"}

Also, tags param is mandatory for this request.
Example tags params:
tags : "tag1,tag2"

This request is to remove tags from an alert in Opsgenie .


function (identifier, data, config, cb) {
            var path = api.getPath(baseURL, identifier, detailsActionPath);
            api.post(path, data, config, cb);
        }

function ( identifier , data , config , cb ) { var path = api . getPath ( baseURL , identifier , detailsActionPath ) ; api . post ( path , data , config , cb ) ; }

This request is to add details as custom properties to an alert in Opsgenie.

One of id, tinyId or alias parameters should be specified with add details request as identifier.
Alias option can only be used for open alerts
example identifier objects:
{identifier : "123", identifierType : "tiny"}
{identifier : "123-23-123", identifierType : "id"}
{identifier : "alertAlias", identifierType : "alias"}

Also, details param is mandatory for this request. This param should specify key-value pairs to be added as extra properties to the alert.
Example details params:
details : { key1 : "value1", key2 : "value2" }

This request is to add details as custom properties to an alert in Opsgenie .


function (identifier, data, config, cb) {
            if (data.action === undefined) {
                return new Error("action param can not be empty.");
            }

            var path = api.getPath(baseURL, identifier, customActionPath + data.action);
            api.post(path, data, config, cb);
        }

function
(
identifier
,
data
,
config
,
cb
)
{
if
(
data
.
action
===
undefined
)
{
return
new
Error
(
"action param can not be empty."
)
;
}
var
path
=
api
.
getPath
(
baseURL
,
identifier
,
customActionPath
+
data
.
action
)
;
api
.
post
(
path
,
data
,
config
,
cb
)
;
}

This request is to execute a custom action on an alert in Opsgenie.

One of id, tinyId or alias parameters should be specified with execute custom action request as identifier.
Alias option can only be used for open alerts
example identifier objects:
{identifier : "123", identifierType : "tiny"}
{identifier : "123-23-123", identifierType : "id"}
{identifier : "alertAlias", identifierType : "alias"}

Also, action param is mandatory for this request. This param should specify the name of the custom action.
Example action param:
action : "actionName"

This request is to execute a custom action on an alert in Opsgenie .


function (identifier, data, config, cb) {
            var path = api.getPathWithData(baseURL, identifier, listLogsPath, data);
            api.get(path, config, cb);
        }

function ( identifier , data , config , cb ) { var path = api . getPathWithData ( baseURL , identifier , listLogsPath , data ) ; api . get ( path , config , cb ) ; }

This request is to list logs of an alert in Opsgenie.

One of id, tinyId or alias parameters should be specified with list alert logs request as identifier.
Alias option can only be used for open alerts
example identifier objects:
{identifier : "123", identifierType : "tiny"}
{identifier : "123-23-123", identifierType : "id"}
{identifier : "alertAlias", identifierType : "alias"}

Also, extra query parameters can be provided to customize the result of this request. See docs for further details:
https://www.opsgenie.com/docs/rest-api/alert-api#listAlertLogs

You can also check /samples/alertV2/listLogs.js for an example.

This request is to list logs of an alert in Opsgenie .


function (identifier, data, config, cb) {
            var path = api.getPath(savedSearchbaseURL, identifier, null);
            api.patch(path, data, config, cb)
        }

function ( identifier , data , config , cb ) { var path = api . getPath ( savedSearchbaseURL , identifier , null ) ; api . patch ( path , data , config , cb ) }

This request is to update a saved search for searching alerts in Opsgenie.

This request is to update a saved search for searching alerts in Opsgenie .


async function optimize(file) {
    check(file);

    const name = getName(file);

    log('reading file ' + path.basename(name));

    const data = await readFile(name, 'utf8');
    return onDataRead(file, data);
}

async
function
optimize
(
file
)
{
check
(
file
)
;
const
name
=
getName
(
file
)
;
log
(
'reading file '
+
path
.
basename
(
name
)
)
;
const
data
=
await
readFile
(
name
,
'utf8'
)
;
return
onDataRead
(
file
,
data
)
;
}

function minificate js,css and html files

@param files     -   js, css or html file path

function minificate js css and html files


async function onDataRead(filename, data) {
    log('file ' + path.basename(filename) + ' read');

    const ext = path.extname(filename).replace(/^\./, '');

    const optimizedData = await minify[ext](data);

    let b64Optimize;
    if (ext === 'css')
        [, b64Optimize] = await tryToCatch(minify.img, filename, optimizedData);

    return b64Optimize || optimizedData;
}

async
function
onDataRead
(
filename
,
data
)
{
log
(
'file '
+
path
.
basename
(
filename
)
+
' read'
)
;
const
ext
=
path
.
extname
(
filename
)
.
replace
(
/
^\.
/
,
''
)
;
const
optimizedData
=
await
minify
[
ext
]
(
data
)
;
let
b64Optimize
;
if
(
ext
===
'css'
)
[
,
b64Optimize
]
=
await
tryToCatch
(
minify
.
img
,
filename
,
optimizedData
)
;
return
b64Optimize
||
optimizedData
;
}

Processing of files
@param fileData {name, data}


function applyOptions(options) {
  const settings = {};
  options = options || {};
  Object.assign(settings, defaults, options);

  return settings;
}

function applyOptions ( options ) { const settings = { } ; options = options || { } ; Object . assign ( settings , defaults , options ) ; return settings ; }

Merge default options with overwrites and returns new object


function parseForwarded(value) {
  const forwarded = {}

  value.trim().split(';').forEach((part) => {
    const pair = part.trim().split('=');
    forwarded[pair[0]] = pair[1];
  });

  return forwarded;
}

function parseForwarded ( value ) { const forwarded = { } value . trim ( ) . split ( ';' ) . forEach ( ( part ) => { const pair = part . trim ( ) . split ( '=' ) ; forwarded [ pair [ 0 ] ] = pair [ 1 ] ; } ) ; return forwarded ; }


function loadRcConfig(callback) {
  const sync = typeof callback !== 'function';

  if (sync) {
    const fp = rcLoader.for(this.resourcePath);
    if (typeof fp !== 'string') {
      // no .jshintrc found
      return {};
    }
    this.addDependency(fp);
    const options = loadConfig(fp);
    delete options.dirname;
    return options;
  }

  // eslint-disable-next-line consistent-return
  rcLoader.for(this.resourcePath, (err, fp) => {
    if (typeof fp !== 'string') {
      // no .jshintrc found
      return callback(null, {});
    }

    this.addDependency(fp);
    const options = loadConfig(fp);
    delete options.dirname;
    callback(err, options);
  });
}

eslint - disable - next - line consistent - return


function inferMaster() {
      var inferred = _.find(self.nestedLocales, { name: self.defaultLocale }) || self.nestedLocales[0];
      if ((self.nestedLocales.length > 1) && (!_.find(self.nestedLocales, function(locale) {
        return locale.children && locale.children.length;
      }))) {
        var others = _.filter(self.nestedLocales, function(locale) {
          return locale.name !== inferred.name;
        });
        self.nestedLocales = [ inferred ];
        if (others.length) {
          inferred.children = others;
        }
      }
    }

function inferMaster ( ) { var inferred = _ . find ( self . nestedLocales , { name : self . defaultLocale } ) || self . nestedLocales [ 0 ] ; if ( ( self . nestedLocales . length > 1 ) && ( ! _ . find ( self . nestedLocales , function ( locale ) { return locale . children && locale . children . length ; } ) ) ) { var others = _ . filter ( self . nestedLocales , function ( locale ) { return locale . name !== inferred . name ; } ) ; self . nestedLocales = [ inferred ] ; if ( others . length ) { inferred . children = others ; } } }

If there is no locale hierarchy, infer that the default locale is the master, or the first locale if there is no default. This is a bc gesture, master locales are a best practice

If there is no locale hierarchy infer that the default locale is the master or the first locale if there is no default . This is a bc gesture master locales are a best practice


function workflowGuidWorkflowLocale() {
      return self.apos.docs.db.ensureIndex({ workflowGuid: 1, workflowLocale: 1 }, { sparse: 1, unique: 1 })
        // eslint-disable-next-line handle-callback-err
        .catch(function(err) {
          return resolveDuplicateDocs()
            .then(function() {
              // now we can try the index again, recursively
              return workflowGuidWorkflowLocale();
            });
        });
    }

Early versions did not have this index and thus could violate the uniqueness of the guid.locale pair under rare race conditions. If the index fails, resolve the duplicates and try again

Early versions did not have this index and thus could violate the uniqueness of the guid . locale pair under rare race conditions . If the index fails resolve the duplicates and try again


function replaceIdsRecursively(doc) {
                _.each(doc, function(val, key) {
                  if (key === 'workflowGuidAndLocaleDuplicates') {
                  // Do not alter ids inside the historical archive of
                  // the original orphaned docs. -Tom
                    return;
                  }
                  if ((typeof (val) === 'string') && (val.length < 100)) {
                    if (idsToNew[val]) {
                      self.apos.utils.warn('Correcting ' + doc[key] + ' to ' + idsToNew[val] + ' in join');
                      doc[key] = idsToNew[val];
                      modified = true;
                    }
                  } else if (val && (typeof (val) === 'object')) {
                    replaceIdsRecursively(val);
                  }
                });
              }

Recursively replace all occurrences of the ids in this locale found in the given doc with their new ids per `idsToNew`. This prevents _id conflicts on insert, even though old data is still in the database under other locale names

Recursively replace all occurrences of the ids in this locale found in the given doc with their new ids per idsToNew . This prevents _id conflicts on insert even though old data is still in the database under other locale names


function fallback(callback) {

      return async.series([ viaBasis, noBasis ], callback);

      function viaBasis(callback) {
        return self.apos.migrations.eachDoc({ workflowLocale: basis }, 5, function(doc, callback) {
          if (!self.includeType(doc.type)) {
            return setImmediate(callback);
          }
          doc.workflowResolveDeferred = true;
          var afterSaveOptions = _.assign({}, options, {
            permissions: false,
            workflowMissingLocalesLive: self.apos.argv.live ? true : 'liveOnly'
          });
          return self.docAfterSave(req, doc, afterSaveOptions, callback);
        }, callback);
      }

      function noBasis(callback) {
        var localeNames = Object.keys(self.locales);
        var orphans;
        return async.series([ find, fix ], callback);
        function find(callback) {
          const query = [
            {
              $match: {
                workflowLocale: { $in: localeNames }
              }
            },
            {
              $group: {
                _id: "$workflowGuid",
                count: { $sum: 1 }
              }
            },
            {
              $match: {
                count: { $lt: localeNames.length }
              }
            }
          ];

          return self.apos.docs.db.aggregate(query).toArray(function(err, _orphans) {
            if (err) {
              return callback(err);
            }
            orphans = _orphans;
            return callback(null);
          });
        }

        function fix(callback) {
          var seen = {};
          if (!orphans.length) {
            return callback(null);
          }
          // The aggregation query returns the workflowGuids but I haven't been able to
          // convince it to give me representative _ids to go with them, so we will just
          // ignore the additional locales after we fix each one once. -Tom
          return self.apos.migrations.eachDoc({ workflowGuid: { $in: _.pluck(orphans, '_id') } }, 5, function(doc, callback) {
            if (seen[doc.workflowGuid]) {
              return setImmediate(callback);
            }
            seen[doc.workflowGuid] = true;
            if (!self.includeType(doc.type)) {
              return setImmediate(callback);
            }
            doc.workflowResolveDeferred = true;
            var afterSaveOptions = _.assign({}, options, {
              permissions: false,
              workflowMissingLocalesLive: self.apos.argv.live ? true : 'liveOnly'
            });
            return self.docAfterSave(req, doc, afterSaveOptions, callback);
          }, callback);
        }
      }

    }

There are cases where a suitable ancestor in the tree cannot be found, or a few docs are missing due to race conditions. To mop these up, we make a penultimate pass that considers every unique workflowGuid, preferring the locale with the most docs as a basis but falling back if necessary to any in which it exists. And in the final pass, we use an aggregation query to find absolutely every last workflowGuid that does not have every locale.

There are cases where a suitable ancestor in the tree cannot be found or a few docs are missing due to race conditions . To mop these up we make a penultimate pass that considers every unique workflowGuid preferring the locale with the most docs as a basis but falling back if necessary to any in which it exists . And in the final pass we use an aggregation query to find absolutely every last workflowGuid that does not have every locale .


function getWorkflowGuids(callback) {
        return self.apos.docs.db.findWithProjection(
          { _id: { $in: _.pluck(related, '_id') } },
          { _id: 1, workflowGuid: 1, type: 1 }
        ).toArray(function(err, guidDocs) {
          if (err) {
            return fail(err);
          }
          _.each(guidDocs, function(guidDoc) {
            var relatedOne = _.find(related, { _id: guidDoc._id });
            if (relatedOne) {
              relatedOne.workflowGuid = guidDoc.workflowGuid;
              idsByGuid[relatedOne.workflowGuid] = relatedOne._id;
              relatedOne.type = guidDoc.type;
            }
          });
          // Discard unless the type is known, the type is subject to
          // workflow and we have the workflowGuid
          related = _.filter(related, function(relatedOne) {
            return relatedOne.type && relatedOne.workflowGuid && self.includeType(relatedOne.type);
          });
          return callback(null);
        });
      }

Due to join projections we often do not know the guids of the related docs yet, and even sometimes the type as well

Due to join projections we often do not know the guids of the related docs yet and even sometimes the type as well


function toAwsParams(file) {
  var params = {};

  var headers = file.s3.headers || {};

  for (var header in headers) {
    if (header === 'x-amz-acl') {
      params.ACL = headers[header];
    } else if (header === 'Content-MD5') {
      params.ContentMD5 = headers[header];
    } else {
      params[pascalCase(header)] = headers[header];
    }
  }

  params.Key = file.s3.path;
  params.Body = file.contents;

  return params;
}

function toAwsParams ( file ) { var params = { } ; var headers = file . s3 . headers || { } ; for ( var header in headers ) { if ( header === 'x-amz-acl' ) { params . ACL = headers [ header ] ; } else if ( header === 'Content-MD5' ) { params . ContentMD5 = headers [ header ] ; } else { params [ pascalCase ( header ) ] = headers [ header ] ; } } params . Key = file . s3 . path ; params . Body = file . contents ; return params ; }

Turn the HTTP style headers into AWS Object params


function Publisher(AWSConfig, cacheOptions) {
  this.config = AWSConfig;
  this.client = new AWS.S3(AWSConfig);
  var bucket = this.config.params.Bucket;

  if (!bucket) {
    throw new Error('Missing `params.Bucket` config value.');
  }

  // init Cache file
  this._cacheFile =
    cacheOptions && cacheOptions.cacheFileName
      ? cacheOptions.cacheFileName
      : '.awspublish-' + bucket;

  // load cache
  try {
    this._cache = JSON.parse(fs.readFileSync(this.getCacheFilename(), 'utf8'));
  } catch (err) {
    this._cache = {};
  }
}

function
Publisher
(
AWSConfig
,
cacheOptions
)
{
this
.
config
=
AWSConfig
;
this
.
client
=
new
AWS
.
S3
(
AWSConfig
)
;
var
bucket
=
this
.
config
.
params
.
Bucket
;
if
(
!
bucket
)
{
throw
new
Error
(
'Missing `params.Bucket` config value.'
)
;
}
// init Cache file
this
.
_cacheFile
=
cacheOptions
&&
cacheOptions
.
cacheFileName
cacheOptions
.
cacheFileName
:
'.awspublish-'
+
bucket
;
// load cache
try
{
this
.
_cache
=
JSON
.
parse
(
fs
.
readFileSync
(
this
.
getCacheFilename
(
)
,
'utf8'
)
)
;
}
catch
(
err
)
{
this
.
_cache
=
{
}
;
}
}

create a new Publisher
@param {Object} S3 options as per http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
@api private


function ZarinPal(MerchantID, sandbox) {
	if (typeof MerchantID === 'string' && MerchantID.length === config.merchantIDLength) {
		this.merchant = MerchantID;
	} else {
		console.error('The MerchantID must be ' + config.merchantIDLength + ' Characters.');
		return false;
	}
	this.sandbox = sandbox || false;

	this.url = (sandbox === true) ? config.sandbox : config.https;
}

function
ZarinPal
(
MerchantID
,
sandbox
)
{
if
(
typeof
MerchantID
===
'string'
&&
MerchantID
.
length
===
config
.
merchantIDLength
)
{
this
.
merchant
=
MerchantID
;
}
else
{
console
.
error
(
'The MerchantID must be '
+
config
.
merchantIDLength
+
' Characters.'
)
;
return
false
;
}
this
.
sandbox
=
sandbox
||
false
;
this
.
url
=
(
sandbox
===
true
)
config
.
sandbox
:
config
.
https
;
}

Constructor for ZarinPal object.
@param {String} MerchantID
@param {bool} sandbox


function deferred() {
    var outResolve = function() {}; // will be overwritten
    var outReject = function() {}; // will be overwritten
    var promise = new Promise(function (resolve, reject) {
        outResolve = resolve;
        outReject = reject;
    });
    return {
        promise: promise,
        resolve: outResolve,
        reject: outReject,
    };
}

function
deferred
(
)
{
var
outResolve
=
function
(
)
{
}
;
// will be overwritten
var
outReject
=
function
(
)
{
}
;
// will be overwritten
var
promise
=
new
Promise
(
function
(
resolve
,
reject
)
{
outResolve
=
resolve
;
outReject
=
reject
;
}
)
;
return
{
promise
:
promise
,
resolve
:
outResolve
,
reject
:
outReject
,
}
;
}


async function parseResponse(header, response, active = false, id) {
    const rows = [];
    _.each(response, (value, key) => {
        let row = [];
        if (active) {
            _.each(response[key].active, (activeValue) => {
                row = [];
                // filter by id
                if (id === undefined || activeValue.job_id === id || activeValue.ex_id === id) {
                    _.each(header, (item) => {
                        if (item === 'teraslice_version') {
                            row.push(response[key].teraslice_version);
                        } else if (item === 'node_id') {
                            row.push(response[key].node_id);
                        } else if (item === 'hostname') {
                            row.push(response[key].hostname);
                        } else if (item === 'node_version') {
                            row.push(response[key].node_version);
                        } else {
                            row.push(activeValue[item]);
                        }
                    });
                    rows.push(row);
                }
            });
        } else {
            _.each(header, (item) => {
                if (item === 'active') {
                    row.push(response[key][item].length);
                } else {
                    row.push(response[key][item]);
                }
            });
            rows.push(row);
        }
    });
    return rows;
}

async
function
parseResponse
(
header
,
response
,
active
=
false
,
id
)
{
const
rows
=
[
]
;
_
.
each
(
response
,
(
value
,
key
)
=>
{
let
row
=
[
]
;
if
(
active
)
{
_
.
each
(
response
[
key
]
.
active
,
(
activeValue
)
=>
{
row
=
[
]
;
// filter by id
if
(
id
===
undefined
||
activeValue
.
job_id
===
id
||
activeValue
.
ex_id
===
id
)
{
_
.
each
(
header
,
(
item
)
=>
{
if
(
item
===
'teraslice_version'
)
{
row
.
push
(
response
[
key
]
.
teraslice_version
)
;
}
else
if
(
item
===
'node_id'
)
{
row
.
push
(
response
[
key
]
.
node_id
)
;
}
else
if
(
item
===
'hostname'
)
{
row
.
push
(
response
[
key
]
.
hostname
)
;
}
else
if
(
item
===
'node_version'
)
{
row
.
push
(
response
[
key
]
.
node_version
)
;
}
else
{
row
.
push
(
activeValue
[
item
]
)
;
}
}
)
;
rows
.
push
(
row
)
;
}
}
)
;
}
else
{
_
.
each
(
header
,
(
item
)
=>
{
if
(
item
===
'active'
)
{
row
.
push
(
response
[
key
]
[
item
]
.
length
)
;
}
else
{
row
.
push
(
response
[
key
]
[
item
]
)
;
}
}
)
;
rows
.
push
(
row
)
;
}
}
)
;
return
rows
;
}

Parses the teraslice client endpoint responses into an array used to generate
tables in command line output.

@param {Array} header - Header values to include in output
@param {Object} response - Teraslice client response object
@param {Boolean} active - When set to true parse values in active list
@param {String} id - id value used to filter results by job_id or ex_id
@returns {Array} rows - Table content is an array of arrays with
each row an element in the array.

Parses the teraslice client endpoint responses into an array used to generate tables in command line output .


async function display(header, items, type, active = false, parse = false, id) {
        let rows;
        if (type === 'txt') {
            await text(header, items);
        } else if (type === 'txtHorizontal') {
            const opts = {
                head: header,
                chars: {
                    top: '',
                    'top-mid': '',
                    'top-left': '',
                    'top-right': '',
                    bottom: '',
                    'bottom-mid': '',
                    'bottom-left': '',
                    'bottom-right': '',
                    left: '',
                    'left-mid': '',
                    mid: '',
                    'mid-mid': '',
                    right: '',
                    'right-mid': '',
                    middle: ' '.repeat(2)
                },
                style: {
                    'padding-left': 0, 'padding-right': 0, head: ['white'], border: ['white']
                }
            };
            if (parse) {
                rows = await parseResponse(header, items, active, id);
                await horizontal(rows, opts);
            } else {
                await horizontal(items, opts);
            }
        } else if (type === 'prettyHorizontal') {
            const opts = {
                head: header,
                style: { 'padding-left': 0, 'padding-right': 0, head: ['blue'] }
            };
            if (parse) {
                rows = await parseResponse(header, items, active, id);
                await horizontal(rows, opts);
            } else {
                await horizontal(items, opts);
            }
        } else if (type === 'txtVertical') {
            const style = {
                chars: {
                    top: '',
                    'top-mid': '',
                    'top-left': '',
                    'top-right': '',
                    bottom: '',
                    'bottom-mid': '',
                    'bottom-left': '',
                    'bottom-right': '',
                    left: '',
                    'left-mid': '',
                    mid: '',
                    'mid-mid': '',
                    right: '',
                    'right-mid': '',
                    middle: ' '.repeat(3)
                },
                style: {
                    'padding-left': 0, 'padding-right': 0, head: ['white']
                }
            };
            if (parse) {
                rows = await parseResponse(header, items, active, id);
                await vertical(header, rows, style);
            } else {
                await vertical(header, items, style);
            }
        } else if (type === 'prettyVertical') {
            const style = {
                style: { 'padding-left': 0, 'padding-right': 0, head: ['blue'] }
            };
            if (parse) {
                rows = await parseResponse(header, items, active, id);
                await vertical(header, rows, style);
            } else {
                await vertical(header, items, style);
            }
        } else {
            await pretty(header, items);
        }
    }

async
function
display
(
header
,
items
,
type
,
active
=
false
,
parse
=
false
,
id
)
{
let
rows
;
if
(
type
===
'txt'
)
{
await
text
(
header
,
items
)
;
}
else
if
(
type
===
'txtHorizontal'
)
{
const
opts
=
{
head
:
header
,
chars
:
{
top
:
''
,
'top-mid'
:
''
,
'top-left'
:
''
,
'top-right'
:
''
,
bottom
:
''
,
'bottom-mid'
:
''
,
'bottom-left'
:
''
,
'bottom-right'
:
''
,
left
:
''
,
'left-mid'
:
''
,
mid
:
''
,
'mid-mid'
:
''
,
right
:
''
,
'right-mid'
:
''
,
middle
:
' '
.
repeat
(
2
)
}
,
style
:
{
'padding-left'
:
0
,
'padding-right'
:
0
,
head
:
[
'white'
]
,
border
:
[
'white'
]
}
}
;
if
(
parse
)
{
rows
=
await
parseResponse
(
header
,
items
,
active
,
id
)
;
await
horizontal
(
rows
,
opts
)
;
}
else
{
await
horizontal
(
items
,
opts
)
;
}
}
else
if
(
type
===
'prettyHorizontal'
)
{
const
opts
=
{
head
:
header
,
style
:
{
'padding-left'
:
0
,
'padding-right'
:
0
,
head
:
[
'blue'
]
}
}
;
if
(
parse
)
{
rows
=
await
parseResponse
(
header
,
items
,
active
,
id
)
;
await
horizontal
(
rows
,
opts
)
;
}
else
{
await
horizontal
(
items
,
opts
)
;
}
}
else
if
(
type
===
'txtVertical'
)
{
const
style
=
{
chars
:
{
top
:
''
,
'top-mid'
:
''
,
'top-left'
:
''
,
'top-right'
:
''
,
bottom
:
''
,
'bottom-mid'
:
''
,
'bottom-left'
:
''
,
'bottom-right'
:
''
,
left
:
''
,
'left-mid'
:
''
,
mid
:
''
,
'mid-mid'
:
''
,
right
:
''
,
'right-mid'
:
''
,
middle
:
' '
.
repeat
(
3
)
}
,
style
:
{
'padding-left'
:
0
,
'padding-right'
:
0
,
head
:
[
'white'
]
}
}
;
if
(
parse
)
{
rows
=
await
parseResponse
(
header
,
items
,
active
,
id
)
;
await
vertical
(
header
,
rows
,
style
)
;
}
else
{
await
vertical
(
header
,
items
,
style
)
;
}
}
else
if
(
type
===
'prettyVertical'
)
{
const
style
=
{
style
:
{
'padding-left'
:
0
,
'padding-right'
:
0
,
head
:
[
'blue'
]
}
}
;
if
(
parse
)
{
rows
=
await
parseResponse
(
header
,
items
,
active
,
id
)
;
await
vertical
(
header
,
rows
,
style
)
;
}
else
{
await
vertical
(
header
,
items
,
style
)
;
}
}
else
{
await
pretty
(
header
,
items
)
;
}
}

Display teraslice responses in a table

@param {Array} header - Header values to include in output
@param {Object} response - Teraslice client response object
@param {String} type - Output table type
@param {Boolean} active - Set to true use values in active list
@param {Boolean} parse - Set to true to parse response
@param {String} id - id value used to filter results by job_id or ex_id


function _getClusterState() {
        return k8s.list('app=teraslice', 'pods')
            .then(k8sPods => k8sState.gen(k8sPods, clusterState, clusterNameLabel))
            .catch((err) => {
                // TODO: We might need to do more here.  I think it's OK to just
                // log though.  This only gets used to show slicer info through
                // the API.  We wouldn't want to disrupt the cluster master
                // for rare failures to reach the k8s API.
                logger.error(err, 'Error listing teraslice pods in k8s');
            });
    }

Creates clusterState by iterating over all k8s pods with the label
`app=teraslice`
@constructor
@return      {Promise} [description]

Creates clusterState by iterating over all k8s pods with the label app = teraslice


function allocateWorkers(execution) {
        const kr = new K8sResource(
            'deployments', 'worker', context.sysconfig.teraslice, execution
        );

        const workerDeployment = kr.resource;

        logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);

        return k8s.post(workerDeployment, 'deployment')
            .then(result => logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
            .catch((err) => {
                const error = new TSError(err, {
                    reason: 'Error submitting k8s worker deployment'
                });
                return Promise.reject(error);
            });
    }

Creates k8s deployment that executes Teraslice workers for specified
Execution.
@param  {Object} execution  Object that contains information of Execution
@return {Promise}           [description]

Creates k8s deployment that executes Teraslice workers for specified Execution .


function getChunk(readerClient, slice, opConfig, logger, metadata) {
    const delimiter = opConfig.line_delimiter;

    async function getMargin(offset, length) {
        let margin = '';
        return new Promise(async (resolve) => {
            while (margin.indexOf(delimiter) === -1) {
                // reader clients must return false-y when nothing more to read.
                const chunk = await readerClient(offset, length); // eslint-disable-line no-await-in-loop, max-len
                if (!chunk) {
                    resolve(margin.split(delimiter)[0]);
                    return;
                }
                margin += chunk;
                offset += length; // eslint-disable-line no-param-reassign, max-len
            }
            // Don't read too far - next slice will get it.
            resolve(margin.split(delimiter)[0]);
        });
    }

    let needMargin = false;
    if (slice.length) {
        // Determines whether or not to grab the extra margin.
        if (slice.offset + slice.length !== slice.total) {
            needMargin = true;
        }
    }

    return readerClient(slice.offset, slice.length)
        .then(async (data) => {
            if (data.endsWith(delimiter)) {
                // Skip the margin if the raw data ends with the delimiter since
                // it will end with a complete record.
                needMargin = false;
            }
            if (needMargin) {
                // Want to minimize reads since will typically be over the
                // network. Using twice the average record size as a heuristic.
                const avgSize = _averageRecordSize(data.split(delimiter));
                data += await getMargin(slice.offset + slice.length, 2 * avgSize); // eslint-disable-line no-param-reassign, max-len
            }
            return data;
        })
        .then(data => chunkFormatter[opConfig.format](data, logger, opConfig, metadata, slice));
}

This function will grab the chunk of data specified by the slice plus an extra margin if the slice does not end with the delimiter.

This function will grab the chunk of data specified by the slice plus an extra margin if the slice does not end with the delimiter .


function finishExecution(exId, err) {
        if (err) {
            const error = new TSError(err, {
                reason: `terminal error for execution: ${exId}, shutting down execution`,
                context: {
                    ex_id: exId,
                }
            });
            logger.error(error);
        }
        return getExecutionContext(exId)
            .then((execution) => {
                const status = execution._status;
                if (['stopping', 'stopped'].includes(status)) {
                    logger.debug(`execution ${exId} is already stopping which means there is no need to stop the execution`);
                    return true;
                }

                logger.debug(`execution ${exId} finished, shutting down execution`);
                return clusterService.stopExecution(exId).catch((stopErr) => {
                    const stopError = new TSError(stopErr, {
                        reason: 'error finishing the execution',
                        context: {
                            ex_id: exId,
                        }
                    });
                    logger.error(stopError);
                });
            });
    }

function
finishExecution
(
exId
,
err
)
{
if
(
err
)
{
const
error
=
new
TSError
(
err
,
{
reason
:
`
${
exId
}
`
,
context
:
{
ex_id
:
exId
,
}
}
)
;
logger
.
error
(
error
)
;
}
return
getExecutionContext
(
exId
)
.
then
(
(
execution
)
=>
{
const
status
=
execution
.
_status
;
if
(
[
'stopping'
,
'stopped'
]
.
includes
(
status
)
)
{
logger
.
debug
(
`
${
exId
}
`
)
;
return
true
;
}
logger
.
debug
(
`
${
exId
}
`
)
;
return
clusterService
.
stopExecution
(
exId
)
.
catch
(
(
stopErr
)
=>
{
const
stopError
=
new
TSError
(
stopErr
,
{
reason
:
'error finishing the execution'
,
context
:
{
ex_id
:
exId
,
}
}
)
;
logger
.
error
(
stopError
)
;
}
)
;
}
)
;
}

safely stop the execution without setting the ex status to stopping or stopped


function getActiveExecution(exId) {
        const str = terminalStatusList().map(state => ` _status:${state} `).join('OR');
        const query = `ex_id: ${exId} NOT (${str.trim()})`;
        return searchExecutionContexts(query, null, 1, '_created:desc')
            .then((ex) => {
                if (ex.length === 0) {
                    const error = new Error(`no active execution context was found for ex_id: ${exId}`);
                    error.code = 404;
                    return Promise.reject(error);
                }
                return ex[0];
            });
    }

function getActiveExecution ( exId ) { const str = terminalStatusList ( ) . map ( state => ` ${ state } ` ) . join ( 'OR' ) ; const query = ` ${ exId } ${ str . trim ( ) } ` ; return searchExecutionContexts ( query , null , 1 , '_created:desc' ) . then ( ( ex ) => { if ( ex . length === 0 ) { const error = new Error ( ` ${ exId } ` ) ; error . code = 404 ; return Promise . reject ( error ) ; } return ex [ 0 ] ; } ) ; }

encompasses all executions in either initialization or running statuses


function gen(k8sPods, clusterState, clusterNameLabel) {
    // Make sure we clean up the old
    const hostIPs = _.uniq(_.map(k8sPods.items, 'status.hostIP'));
    const oldHostIps = _.difference(_.keys(clusterState), hostIPs);
    _.forEach(oldHostIps, (ip) => {
        delete clusterState[ip];
    });

    // Loop over the nodes in clusterState and set active = [] so we can append
    // later
    Object.keys(clusterState).forEach((nodeId) => {
        clusterState[nodeId].active = [];
    });

    // add a worker for each pod
    k8sPods.items.forEach((pod) => {
        // Teraslice workers and execution controllers have the `clusterName`
        // label that matches their cluster name attached to their k8s pods.
        // If these labels don't match the supplied `clusterNameLabel`
        // then it is assumed that the pod is not a member of this cluster
        // so it is omitted from clusterState.
        // NOTE: The cluster master will not appear in cluster state if they do
        // not label it with clusterName=clusterNameLabel
        if (pod.metadata.labels.clusterName === clusterNameLabel) {
            if (!_.has(clusterState, pod.status.hostIP)) {
                // If the node isn't in clusterState, add it
                clusterState[pod.status.hostIP] = {
                    node_id: pod.status.hostIP,
                    hostname: pod.status.hostIP,
                    pid: 'N/A',
                    node_version: 'N/A',
                    teraslice_version: 'N/A',
                    total: 'N/A',
                    state: 'connected',
                    available: 'N/A',
                    active: []
                };
            }

            const worker = {
                assets: [],
                assignment: pod.metadata.labels.nodeType,
                ex_id: pod.metadata.labels.exId,
                // WARNING: This makes the assumption that the first container
                // in the pod is the teraslice container.  Currently it is the
                // only container, so this assumption is safe for now.
                image: pod.spec.containers[0].image,
                job_id: pod.metadata.labels.jobId,
                pod_name: pod.metadata.name,
                pod_ip: pod.status.podIP,
                worker_id: pod.metadata.name,
            };

            // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`,
            // `Failed`, `Unknown`.  We will only add `Running` pods to the
            // Teraslice cluster state.
            if (pod.status.phase === 'Running') {
                clusterState[pod.status.hostIP].active.push(worker);
            }
        }
    });
}

Given the k8s Pods API output generates the appropriate Teraslice cluster
state
@param  {Object} k8sPods          k8s pods API object (k8s v1.10)
@param  {Object} clusterState     Teraslice Cluster State
@param  {String} clusterNameLabel k8s label containing clusterName


function errorHandler(err) {
        // eslint-disable-next-line no-console
        const logErr = logger ? logger.error.bind(logger) : console.log;
        if (cluster.isMaster) {
            logErr(`Error in master with pid: ${process.pid}`);
        } else {
            logErr(`Error in worker: ${cluster.worker.id} pid: ${process.pid}`);
        }

        if (err.message) {
            logErr(err.message);
        } else {
            logErr(err);
        }

        if (err.stack) {
            logErr(err.stack);
        }

        // log saving to disk is async, using hack to give time to flush
        setTimeout(() => {
            process.exit(-1);
        }, 600);
    }


function allocateWorkers(execution, numOfWorkersRequested) {
        const exId = execution.ex_id;
        const jobId = execution.job_id;
        const jobStr = JSON.stringify(execution);
        const sortedNodes = _.orderBy(clusterState, 'available', 'desc');
        let workersRequested = numOfWorkersRequested;
        let availWorkers = _availableWorkers(false, true);

        const dispatch = _makeDispatch();

        while (workersRequested > 0 && availWorkers > 0) {
            for (let i = 0; i < sortedNodes.length; i += 1) {
                // each iteration check if it can allocate
                if (workersRequested > 0 && availWorkers > 0) {
                    if (sortedNodes[i].available >= 1) {
                        dispatch.set(sortedNodes[i].node_id, 1);
                        availWorkers -= 1;
                        workersRequested -= 1;
                    }
                } else {
                    break;
                }
            }
        }
        // if left over worker requests, enqueue them, queue works based off of id
        // so it redundantly references ex_id

        const workerData = {
            job: jobStr,
            id: exId,
            ex_id: exId,
            job_id: jobId,
            workers: 1,
            assignment: 'worker'
        };

        while (workersRequested > 0) {
            logger.trace(`adding worker to pending queue for ex: ${exId}`);
            pendingWorkerRequests.enqueue(workerData);
            workersRequested -= 1;
        }
        const results = [];

        _.forOwn(dispatch.getDispatch(), (workerCount, nodeId) => {
            const requestedWorkersData = {
                job: jobStr,
                id: exId,
                ex_id: exId,
                job_id: jobId,
                workers: workerCount,
                assignment: 'worker'
            };

            const createRequest = messaging.send({
                to: 'node_master',
                address: nodeId,
                message: 'cluster:workers:create',
                payload: requestedWorkersData,
                response: true
            }).then((msg) => {
                const createdWorkers = _.get(msg, 'payload.createdWorkers');
                if (!_.isInteger(createdWorkers)) {
                    logger.error(`malformed response from create workers request to node ${nodeId}`, msg);
                    return;
                }
                if (createdWorkers < workerCount) {
                    logger.warn(`node ${nodeId} was only able to allocate ${createdWorkers} the request worker count of ${workerCount}, enqueing the remainder`);
                    const newWorkersRequest = _.cloneDeep(requestedWorkersData);
                    newWorkersRequest.workers = workerCount - createdWorkers;
                    pendingWorkerRequests.enqueue(newWorkersRequest);
                } else {
                    logger.debug(`node ${nodeId} allocated ${createdWorkers}`);
                }
            }).catch((err) => {
                logger.error(err, `An error has occurred in allocating : ${workerCount} workers to node : ${nodeId}, the worker request has been enqueued`);
                pendingWorkerRequests.enqueue(requestedWorkersData);
            });

            results.push(createRequest);
        });

        // this will resolve successfully if one worker was actually allocated
        return Promise.all(results);
    }

designed to allocate additional workers, not any future slicers

designed to allocate additional workers not any future slicers


function getRecord(recordId, indexArg, fields) {
        logger.trace(`getting record id: ${recordId}`);
        const query = {
            index: indexArg || indexName,
            type: recordType,
            id: recordId
        };

        if (fields) {
            query._source = fields;
        }
        return elasticsearch.get(query);
    }

function getRecord ( recordId , indexArg , fields ) { logger . trace ( ` ${ recordId } ` ) ; const query = { index : indexArg || indexName , type : recordType , id : recordId } ; if ( fields ) { query . _source = fields ; } return elasticsearch . get ( query ) ; }


function verifyStatusUpdate(exId, desiredStatus) {
        if (!desiredStatus || !_isValidStatus(desiredStatus)) {
            const error = new Error(`Invalid Job status: "${desiredStatus}"`);
            error.statusCode = 422;
            return Promise.reject(error);
        }

        return getStatus(exId)
            .then((status) => {
                // when setting the same status to shouldn't throw an error
                if (desiredStatus === status) {
                    return Promise.resolve();
                }

                // when the current status is running it cannot be set to an init status
                if (_isRunningStatus(status) && _isInitStatus(desiredStatus)) {
                    const error = new Error(`Cannot update running job status of "${status}" to init status of "${desiredStatus}"`);
                    return Promise.reject(error);
                }

                // when the status is a terminal status, it cannot be set to again
                if (_isTerminalStatus(status)) {
                    const error = new Error(`Cannot update terminal job status of "${status}" to "${desiredStatus}"`);
                    return Promise.reject(error);
                }

                // otherwise allow the update
                return Promise.resolve(status);
            });
    }

verify the current status to make sure it can be updated to the desired status


function _attachRoomsSocketIO() {
        if (!io) return;

        // middleware
        io.use((socket, next) => {
            const {
                node_id: nodeId,
            } = socket.handshake.query;

            if (nodeId) {
                logger.info(`node ${nodeId} joining room on connect`);
                socket.join(nodeId);
            }

            return next();
        });
    }

function
_attachRoomsSocketIO
(
)
{
if
(
!
io
)
return
;
// middleware
io
.
use
(
(
socket
,
next
)
=>
{
const
{
node_id
:
nodeId
,
}
=
socket
.
handshake
.
query
;
if
(
nodeId
)
{
logger
.
info
(
`
${
nodeId
}
`
)
;
socket
.
join
(
nodeId
)
;
}
return
next
(
)
;
}
)
;
}

join rooms before on connect to avoid race conditions


function _toRecords(rawData, delimiter, slice) {
    // Since slices with a non-zero chunk offset grab the character
    // immediately preceding the main chunk, if one of those chunks has a
    // delimiter as the first or second character, it means the chunk starts
    // with a complete record. In this case as well as when the chunk begins
    // with a partial record, splitting the chunk into an array by its
    // delimiter will result in a single garbage record at the beginning of
    // the array. If the offset is 0, the array will never start with a
    // garbage record

    let outputData = rawData;
    if (rawData.endsWith(delimiter)) {
        // Get rid of last character if it is the delimiter since that will
        // just result in an empty record.
        outputData = rawData.slice(0, -delimiter.length);
    }

    if (slice.offset === 0) {
        return outputData.split(delimiter);
    }
    return outputData.split(delimiter).splice(1);
}

This function takes the raw data and breaks it into records, getting rid of anything preceding the first complete record if the data does not start with a complete record.

This function takes the raw data and breaks it into records getting rid of anything preceding the first complete record if the data does not start with a complete record .


function raw(incomingData, logger, opConfig, metadata, slice) {
    const data = _toRecords(incomingData, opConfig.line_delimiter, slice);
    return data.map((record) => {
        try {
            return DataEntity.make(
                { data: record },
                metadata
            );
        } catch (err) {
            if (opConfig._dead_letter_action === 'log') {
                logger.error(err, 'Bad record:', record);
            } else if (opConfig._dead_letter_action === 'throw') {
                throw err;
            }
            return null;
        }
    });
}

function
raw
(
incomingData
,
logger
,
opConfig
,
metadata
,
slice
)
{
const
data
=
_toRecords
(
incomingData
,
opConfig
.
line_delimiter
,
slice
)
;
return
data
.
map
(
(
record
)
=>
{
try
{
return
DataEntity
.
make
(
{
data
:
record
}
,
metadata
)
;
}
catch
(
err
)
{
if
(
opConfig
.
_dead_letter_action
===
'log'
)
{
logger
.
error
(
err
,
'Bad record:'
,
record
)
;
}
else
if
(
opConfig
.
_dead_letter_action
===
'throw'
)
{
throw
err
;
}
return
;
}
}
)
;
}

No parsing, leaving to reader or a downstream op.

No parsing leaving to reader or a downstream op .


function tsv(incomingData, logger, opConfig, metadata, slice) {
    return csv(incomingData, logger, opConfig, metadata, slice);
}

function tsv ( incomingData , logger , opConfig , metadata , slice ) { return csv ( incomingData , logger , opConfig , metadata , slice ) ; }


function TransformStreamCloseReadable(transformStream) {
  // console.log('TransformStreamCloseReadable()');

  if (transformStream._errored === true) {
    throw new TypeError('TransformStream is already errored');
  }

  if (transformStream._readableClosed === true) {
    throw new TypeError('Readable side is already closed');
  }

  TransformStreamCloseReadableInternal(transformStream);
}

Methods on the transform stream controller object


function TransformStreamCloseReadableInternal(transformStream) {
  assert(transformStream._errored === false);
  assert(transformStream._readableClosed === false);

  try {
    ReadableStreamDefaultControllerClose(transformStream._readableController);
  } catch (e) {
    assert(false);
  }

  transformStream._readableClosed = true;
}

function TransformStreamCloseReadableInternal ( transformStream ) { assert ( transformStream . _errored === false ) ; assert ( transformStream . _readableClosed === false ) ; try { ReadableStreamDefaultControllerClose ( transformStream . _readableController ) ; } catch ( e ) { assert ( false ) ; } transformStream . _readableClosed = true ; }


function pipeLoop() {
        currentWrite = Promise.resolve();

        if (shuttingDown === true) {
          return Promise.resolve();
        }

        return writer._readyPromise.then(() => {
          return ReadableStreamDefaultReaderRead(reader).then(({ value, done }) => {
            if (done === true) {
              return undefined;
            }

            currentWrite = WritableStreamDefaultWriterWrite(writer, value);
            return currentWrite;
          });
        })
        .then(pipeLoop);
      }

function pipeLoop ( ) { currentWrite = Promise . resolve ( ) ; if ( shuttingDown === true ) { return Promise . resolve ( ) ; } return writer . _readyPromise . then ( ( ) => { return ReadableStreamDefaultReaderRead ( reader ) . then ( ( { value , done } ) => { if ( done === true ) { return undefined ; } currentWrite = WritableStreamDefaultWriterWrite ( writer , value ) ; return currentWrite ; } ) ; } ) . then ( pipeLoop ) ; }

Using reader and writer, read all chunks from this and write them to dest - Backpressure must be enforced - Shutdown must stop all activity

Using reader and writer read all chunks from this and write them to dest - Backpressure must be enforced - Shutdown must stop all activity


function ReadableStreamAddReadIntoRequest(stream) {
  assert(IsReadableStreamBYOBReader(stream._reader) === true);
  assert(stream._state === 'readable' || stream._state === 'closed');

  const promise = new Promise((resolve, reject) => {
    const readIntoRequest = {
      _resolve: resolve,
      _reject: reject
    };

    stream._reader._readIntoRequests.push(readIntoRequest);
  });

  return promise;
}

function ReadableStreamAddReadIntoRequest ( stream ) { assert ( IsReadableStreamBYOBReader ( stream . _reader ) === true ) ; assert ( stream . _state === 'readable' || stream . _state === 'closed' ) ; const promise = new Promise ( ( resolve , reject ) => { const readIntoRequest = { _resolve : resolve , _reject : reject } ; stream . _reader . _readIntoRequests . push ( readIntoRequest ) ; } ) ; return promise ; }

ReadableStream API exposed for controllers.

ReadableStream API exposed for controllers .


function ReadableStreamReaderGenericCancel(reader, reason) {
  const stream = reader._ownerReadableStream;
  assert(stream !== undefined);
  return ReadableStreamCancel(stream, reason);
}

function ReadableStreamReaderGenericCancel ( reader , reason ) { const stream = reader . _ownerReadableStream ; assert ( stream !== undefined ) ; return ReadableStreamCancel ( stream , reason ) ; }

A client of ReadableStreamDefaultReader and ReadableStreamBYOBReader may use these functions directly to bypass state check.

A client of ReadableStreamDefaultReader and ReadableStreamBYOBReader may use these functions directly to bypass state check .


function ReadableStreamDefaultControllerClose(controller) {
  const stream = controller._controlledReadableStream;

  assert(controller._closeRequested === false);
  assert(stream._state === 'readable');

  controller._closeRequested = true;

  if (controller._queue.length === 0) {
    ReadableStreamClose(stream);
  }
}

function ReadableStreamDefaultControllerClose ( controller ) { const stream = controller . _controlledReadableStream ; assert ( controller . _closeRequested === false ) ; assert ( stream . _state === 'readable' ) ; controller . _closeRequested = true ; if ( controller . _queue . length === 0 ) { ReadableStreamClose ( stream ) ; } }

A client of ReadableStreamDefaultController may use these functions directly to bypass state check.

A client of ReadableStreamDefaultController may use these functions directly to bypass state check .


function ReadableByteStreamControllerClose(controller) {
  const stream = controller._controlledReadableStream;

  assert(controller._closeRequested === false);
  assert(stream._state === 'readable');

  if (controller._totalQueuedBytes > 0) {
    controller._closeRequested = true;

    return;
  }

  if (controller._pendingPullIntos.length > 0) {
    const firstPendingPullInto = controller._pendingPullIntos[0];
    if (firstPendingPullInto.bytesFilled > 0) {
      const e = new TypeError('Insufficient bytes to fill elements in the given buffer');
      ReadableByteStreamControllerError(controller, e);

      throw e;
    }
  }

  ReadableStreamClose(stream);
}

A client of ReadableByteStreamController may use these functions directly to bypass state check.

A client of ReadableByteStreamController may use these functions directly to bypass state check .


function WritableStreamAddWriteRequest(stream) {
  assert(IsWritableStreamLocked(stream) === true);
  assert(stream._state === 'writable');

  const promise = new Promise((resolve, reject) => {
    const writeRequest = {
      _resolve: resolve,
      _reject: reject
    };

    stream._writeRequests.push(writeRequest);
  });

  return promise;
}

function WritableStreamAddWriteRequest ( stream ) { assert ( IsWritableStreamLocked ( stream ) === true ) ; assert ( stream . _state === 'writable' ) ; const promise = new Promise ( ( resolve , reject ) => { const writeRequest = { _resolve : resolve , _reject : reject } ; stream . _writeRequests . push ( writeRequest ) ; } ) ; return promise ; }

WritableStream API exposed for controllers.

WritableStream API exposed for controllers .


function IsWritableStreamDefaultWriter(x) {
  if (!typeIsObject(x)) {
    return false;
  }

  if (!Object.prototype.hasOwnProperty.call(x, '_ownerWritableStream')) {
    return false;
  }

  return true;
}

function IsWritableStreamDefaultWriter ( x ) { if ( ! typeIsObject ( x ) ) { return false ; } if ( ! Object . prototype . hasOwnProperty . call ( x , '_ownerWritableStream' ) ) { return false ; } return true ; }

Abstract operations for the WritableStreamDefaultWriter.

Abstract operations for the WritableStreamDefaultWriter .


function WritableStreamDefaultWriterAbort(writer, reason) {
  const stream = writer._ownerWritableStream;

  assert(stream !== undefined);

  return WritableStreamAbort(stream, reason);
}

function WritableStreamDefaultWriterAbort ( writer , reason ) { const stream = writer . _ownerWritableStream ; assert ( stream !== undefined ) ; return WritableStreamAbort ( stream , reason ) ; }

A client of WritableStreamDefaultWriter may use these functions directly to bypass state check.

A client of WritableStreamDefaultWriter may use these functions directly to bypass state check .


function WritableStreamDefaultControllerAbort(controller, reason) {
  controller._queue = [];

  const sinkAbortPromise = PromiseInvokeOrFallbackOrNoop(controller._underlyingSink, 'abort', [reason],
                                                         'close', [controller]);
  return sinkAbortPromise.then(() => undefined);
}

function WritableStreamDefaultControllerAbort ( controller , reason ) { controller . _queue = [ ] ; const sinkAbortPromise = PromiseInvokeOrFallbackOrNoop ( controller . _underlyingSink , 'abort' , [ reason ] , 'close' , [ controller ] ) ; return sinkAbortPromise . then ( ( ) => undefined ) ; }

Abstract operations implementing interface required by the WritableStream.

Abstract operations implementing interface required by the WritableStream .


function WritableStreamDefaultControllerError(controller, e) {
  const stream = controller._controlledWritableStream;

  assert(stream._state === 'writable' || stream._state === 'closing');

  WritableStreamError(stream, e);

  controller._queue = [];
}

function WritableStreamDefaultControllerError ( controller , e ) { const stream = controller . _controlledWritableStream ; assert ( stream . _state === 'writable' || stream . _state === 'closing' ) ; WritableStreamError ( stream , e ) ; controller . _queue = [ ] ; }

A client of WritableStreamDefaultController may use these functions directly to bypass state check.

A client of WritableStreamDefaultController may use these functions directly to bypass state check .


function pipe() {
    var _this = this;
    var items = [];
    for (var _i = 0; _i < arguments.length; _i++) {
        items[_i] = arguments[_i];
    }
    var _done = false;
    var targets = items.map(wrap_async_iterable_iterator_1.wrapAsyncIterableIterator);
    var call = function (methodName, value) {
        return new Promise(function (resolve, reject) {
            var remaining = targets.concat();
            var next = function (_a) {
                var value = _a.value, done = _a.done;
                if (!_done) {
                    _done = done;
                }
                // if one piped item finishes, then we need to finish
                if (!remaining.length || _done) {
                    if (methodName === "next") {
                        while (remaining.length) {
                            (remaining.shift().return || (function () { }))();
                        }
                    }
                    return resolve({ value: value, done: done });
                }
                var fn = remaining.shift()[methodName];
                return fn ? fn(value).then(next, reject) : next(value);
            };
            next({ value: value, done: false });
        });
    };
    return _a = {},
        _a[Symbol.asyncIterator] = function () { return _this; },
        _a.next = call.bind(this, "next"),
        _a.return = call.bind(this, "return"),
        _a.throw = call.bind(this, "throw"),
        _a;
    var _a;
}

Pipes yielded data though each iterable in the pipeline

@param {AsyncIterableIterator|IterableIterator} source the first iterable in the pipeline
@param {AsyncIterableIterator|IterableIterator} [through] proceeding iterables to pass yielded data through

@example
import { pipe, through, readAll } from "mesh";

const negate = (values) => pipe(
values,
through(a => -a)
);

const negativeValues = await readAll(negate([1, 2, 3])); // [-1, -2, -3]


function() {
      var uris = [];
      var db = this.getAnalyser().getDatabase();

      function add(classname) {
        var def = db.classInfo[classname];
        uris.push(def.libraryName + ":" + classname.replace(/\./g, "/") + ".js");
      }
      this.__loadDeps.forEach(add);

      return uris;
    }

function ( ) { var uris = [ ] ; var db = this . getAnalyser ( ) . getDatabase ( ) ; function add ( classname ) { var def = db . classInfo [ classname ] ; uris . push ( def . libraryName + ":" + classname . replace ( / \. / g , "/" ) + ".js" ) ; } this . __loadDeps . forEach ( add ) ; return uris ; }

Gets a list of URIs for classes that are required, in load order

@returns {String[]}

Gets a list of URIs for classes that are required in load order


function() {
      var result = {};
      this.__classes.forEach(name => result[name] = true);
      this.__expandClassnames(this.getInclude()).forEach(name => result[name] = true);
      this.__expandClassnames(this.getExclude()).forEach(name => delete result[name]);

      // We sort the result so that we can get a consistent ordering for loading classes, otherwise the order in
      //  which the filing system returns the files can cause classes to be loaded in a lightly different sequence;
      //  that would not cause a problem, except that the build is not 100% repeatable.
      return Object.keys(result).sort();
    }

Returns the classes required for the application
@returns {String[]}


function(value, oldValue) {
      var loader = path.join(__dirname, "loader-" + (this.isBrowserApp() ? "browser" : "server") + ".tmpl.js");
      this.setLoaderTemplate(loader);
      this.setTheme(null);
    }

function ( value , oldValue ) { var loader = path . join ( __dirname , "loader-" + ( this . isBrowserApp ( ) ? "browser" : "server" ) + ".tmpl.js" ) ; this . setLoaderTemplate ( loader ) ; this . setTheme ( null ) ; }


function mkpath(dir, cb) {
  dir = path.normalize(dir);
  var segs = dir.split(path.sep);
  var made = "";
  async.eachSeries(
      segs,
      function(seg, cb) {
        if (made.length || !seg.length) {
          made += "/";
        }
        made += seg;
        fs.exists(made, function(exists) {
          if (!exists) {
            fs.mkdir(made, function(err) {
              if (err && err.code === "EEXIST") {
                err = null;
              }
              cb(err);
            });
            return;
          }
          fs.stat(made, function(err, stat) {
            if (err) {
              cb(err);
            } else if (stat.isDirectory()) {
              cb(null);
            } else {
              cb(new Error("Cannot create " + made + " (in " + dir + ") because it exists and is not a directory", "ENOENT"));
            }
          });
        });
      },
      function(err) {
        cb(err);
      });
}


function() {
      var t = this;

      var p;
      if (!this.__opened) {
        this.__opened = true;

        var resManager = null;
        if (this.isProcessResources()) {
          resManager = new qx.tool.compiler.resources.Manager(this);
        }
        this.__resManager = resManager;
        p = Promise.all(
            [
              util.promisifyThis(t.loadDatabase, t),
              new Promise((resolve, reject) => {
                if (resManager) {
                  log.debug("Loading resource database");
                  return util.promisifyThis(resManager.loadDatabase, resManager)
                    .then(resolve)
                    .catch(reject);
                }
                resolve();
                return undefined;
              })
            ]);
      } else {
        p = Promise.resolve();
      }

      return p.then(() => {
          log.debug("Scanning source code");
          return util.promisifyThis(t.initialScan, t);
        })
        .then(() => {
          log.debug("Saving database");
          return t.saveDatabase();
        });
    }

function
(
)
{
var
t
=
this
;
var
p
;
if
(
!
this
.
__opened
)
{
this
.
__opened
=
true
;
var
resManager
=
;
if
(
this
.
isProcessResources
(
)
)
{
resManager
=
new
qx
.
tool
.
compiler
.
resources
.
Manager
(
this
)
;
}
this
.
__resManager
=
resManager
;
p
=
Promise
.
all
(
[
util
.
promisifyThis
(
t
.
loadDatabase
,
t
)
,
new
Promise
(
(
resolve
,
reject
)
=>
{
if
(
resManager
)
{
log
.
debug
(
"Loading resource database"
)
;
return
util
.
promisifyThis
(
resManager
.
loadDatabase
,
resManager
)
.
then
(
resolve
)
.
catch
(
reject
)
;
}
resolve
(
)
;
return
undefined
;
}
)
]
)
;
}
else
{
p
=
Promise
.
resolve
(
)
;
}
return
p
.
then
(
(
)
=>
{
log
.
debug
(
"Scanning source code"
)
;
return
util
.
promisifyThis
(
t
.
initialScan
,
t
)
;
}
)
.
then
(
(
)
=>
{
log
.
debug
(
"Saving database"
)
;
return
t
.
saveDatabase
(
)
;
}
)
;
}

Opens the analyser, loads database etc

@async


function(cb) {
      var t = this;
      if (!this.__db) {
        this.__db = {};
      }
      async.parallel(
          [
            // Load Resources
            function(cb) {
              if (!t.__resManager) {
                cb(null);
                return;
              }

              t.__resManager.findAllResources(function(err) {
                if (err) {
                  cb(err);
                  return;
                }
                log.debug("found all resources");
                cb(null);
              });
            },

            // Find all classes
            function(cb) {
              async.each(t.__libraries,
                  function(library, cb) {
                    library.scanForClasses(err => {
                      log.debug("Finished scanning for " + library.getNamespace());
                      cb(err);
                    });
                  },
                  err => {
                    log.debug("Finished scanning for all libraries");
                    cb(err);
                  });
            }
          ],
          function(err) {
            log.debug("processed source and resources");
            cb(err);
          });
    }

function
(
cb
)
{
var
t
=
this
;
if
(
!
this
.
__db
)
{
this
.
__db
=
{
}
;
}
async
.
parallel
(
[
// Load Resources
function
(
cb
)
{
if
(
!
t
.
__resManager
)
{
cb
(
)
;
return
;
}
t
.
__resManager
.
findAllResources
(
function
(
err
)
{
if
(
err
)
{
cb
(
err
)
;
return
;
}
log
.
debug
(
"found all resources"
)
;
cb
(
)
;
}
)
;
}
,
// Find all classes
function
(
cb
)
{
async
.
each
(
t
.
__libraries
,
function
(
library
,
cb
)
{
library
.
scanForClasses
(
err
=>
{
log
.
debug
(
"Finished scanning for "
+
library
.
getNamespace
(
)
)
;
cb
(
err
)
;
}
)
;
}
,
err
=>
{
log
.
debug
(
"Finished scanning for all libraries"
)
;
cb
(
err
)
;
}
)
;
}
]
,
function
(
err
)
{
log
.
debug
(
"processed source and resources"
)
;
cb
(
err
)
;
}
)
;
}

Scans the source files for javascript class and resource references and
calculates the dependency tree

@param cb


function(cb) {
              async.each(t.__libraries,
                  function(library, cb) {
                    library.scanForClasses(err => {
                      log.debug("Finished scanning for " + library.getNamespace());
                      cb(err);
                    });
                  },
                  err => {
                    log.debug("Finished scanning for all libraries");
                    cb(err);
                  });
            }

function
(
cb
)
{
async
.
each
(
t
.
__libraries
,
function
(
library
,
cb
)
{
library
.
scanForClasses
(
err
=>
{
log
.
debug
(
"Finished scanning for "
+
library
.
getNamespace
(
)
)
;
cb
(
err
)
;
}
)
;
}
,
err
=>
{
log
.
debug
(
"Finished scanning for all libraries"
)
;
cb
(
err
)
;
}
)
;
}


function(cb) {
      var t = this;
      async.waterfall(
          [
            /**
             * Reads the db.json, if it exists
             *
             * @param cb
             */
            function readDb(cb) {
              fs.exists(t.getDbFilename(), function(exists) {
                if (exists) {
                  fs.readFile(t.getDbFilename(), {encoding: "utf-8"}, cb);
                } else {
                  cb(null, null);
                }
              });
            },

            /**
             * Parses the db.json into db
             *
             * @param data
             * @param cb
             */
            function parseDb(data, cb) {
              if (data && data.trim().length) {
                log.debug("Parsing database");
                t.__db = jsonlint.parse(data);
              } else {
                log.debug("No database to parse");
                t.__db = {};
              }
              cb(null, t.__db);
            }
          ],

          /**
           * Done
           * @param err
           * @param result
           */
          function(err, result) {
            log.debug("loaded database: err=" + err);
            cb();
          });
    }

Loads the database if available

@param cb


function readDb(cb) {
              fs.exists(t.getDbFilename(), function(exists) {
                if (exists) {
                  fs.readFile(t.getDbFilename(), {encoding: "utf-8"}, cb);
                } else {
                  cb(null, null);
                }
              });
            }

function readDb ( cb ) { fs . exists ( t . getDbFilename ( ) , function ( exists ) { if ( exists ) { fs . readFile ( t . getDbFilename ( ) , { encoding : "utf-8" } , cb ) ; } else { cb ( null , null ) ; } } ) ; }

Reads the db.json, if it exists

@param cb


function parseDb(data, cb) {
              if (data && data.trim().length) {
                log.debug("Parsing database");
                t.__db = jsonlint.parse(data);
              } else {
                log.debug("No database to parse");
                t.__db = {};
              }
              cb(null, t.__db);
            }

function
parseDb
(
data
,
cb
)
{
if
(
data
&&
data
.
trim
(
)
.
length
)
{
log
.
debug
(
"Parsing database"
)
;
t
.
__db
=
jsonlint
.
parse
(
data
)
;
}
else
{
log
.
debug
(
"No database to parse"
)
;
t
.
__db
=
{
}
;
}
cb
(
,
t
.
__db
)
;
}

Parses the db.json into db

@param data
@param cb


function(className) {
      let t = this;

      // __classes will be null if analyseClasses has not formally been called; this would be if the
      //  analyser is only called externally for getClass()
      if (!t.__classes) {
        t.__classes = [];
      }

      // Add it
      if (t.__classes.indexOf(className) == -1) {
        t.__classes.push(className);
      }
    }

Called when a reference to a class is made
@param className
@private


async function(locale) {
      var t = this;
      var cldr = this.__cldrs[locale];
      if (cldr) {
        return cldr;
      }
      return qx.tool.compiler.app.Cldr.loadCLDR(locale)
        .then(cldr => t.__cldrs[locale] = cldr);
    }

async function ( locale ) { var t = this ; var cldr = this . __cldrs [ locale ] ; if ( cldr ) { return cldr ; } return qx . tool . compiler . app . Cldr . loadCLDR ( locale ) . then ( cldr => t . __cldrs [ locale ] = cldr ) ; }

Returns the CLDR data for a given locale
@param locale {String} the locale string
@returns Promise({cldr})


async function(library, locale) {
      var t = this;
      var id = locale + ":" + library.getNamespace();
      var translation = t.__translations[id];
      if (!translation) {
        translation = t.__translations[id] = new qx.tool.compiler.app.Translation(library, locale);
        await translation.checkRead();
      }
      return translation;
    }

async function ( library , locale ) { var t = this ; var id = locale + ":" + library . getNamespace ( ) ; var translation = t . __translations [ id ] ; if ( ! translation ) { translation = t . __translations [ id ] = new qx . tool . compiler . app . Translation ( library , locale ) ; await translation . checkRead ( ) ; } return translation ; }

Gets the translation for the locale and library, caching teh result.
@param library
@param locale
@returns {Promise(translation)}

Gets the translation for the locale and library caching teh result .


function(library, locales) {
      const Promisify = qx.tool.compiler.utils.Promisify;

      return Promise.all(locales.map(locale => {
        var translation = new qx.tool.compiler.app.Translation(library, locale);
        return translation.read()
        .then(() => {
          let unusedEntries = {};
          for (let msgid in translation.getEntries())
            unusedEntries[msgid] = true;

          return Promise.all(this.__classes.map(async classname => {
            if (!classname.startsWith(library.getNamespace())) {
              return;
            }

            let dbClassInfo = await Promisify.call(cb => this.getClassInfo(classname, cb));
            if (!dbClassInfo.translations)
              return;

            dbClassInfo.translations.forEach(function(src) {
              var entry = translation.getOrCreateEntry(src.msgid);
              delete unusedEntries[src.msgid];
              if (src.msgid_plural) {
                entry.msgid_plural = src.msgid_plural;
              }
              if (!entry.comments) {
                entry.comments = {};
              }
              entry.comments.extracted = src.comment;
              entry.comments.reference = {};
              let ref = entry.comments.reference;
              const fileName = classname.replace(/\./g, "/") + ".js";
              const fnAddReference = lineNo => {
                let arr = ref[fileName];
                if (!arr)
                  arr = ref[fileName] = [];

                if (!arr.includes(src.lineNo)) {
                  arr.push(lineNo);
                }
              };
              if (qx.lang.Type.isArray(src.lineNo)) {
                src.lineNo.forEach(fnAddReference);
              } else {
                fnAddReference(src.lineNo);
              }
            });
          }))
          .then(() => {
            Object.keys(unusedEntries).forEach(msgid => {
              var entry = translation.getEntry(msgid);
              if (entry) {
                if (!entry.comments) {
                  entry.comments = {};
                }
                entry.comments.extracted = "NO LONGER USED";
                entry.comments.reference = {};
              }
            });
          });
        })
        .then(() => translation.write());
      }));
    }

function
(
library
,
locales
)
{
const
Promisify
=
qx
.
tool
.
compiler
.
utils
.
Promisify
;
return
Promise
.
all
(
locales
.
map
(
locale
=>
{
var
translation
=
new
qx
.
tool
.
compiler
.
app
.
Translation
(
library
,
locale
)
;
return
translation
.
read
(
)
.
then
(
(
)
=>
{
let
unusedEntries
=
{
}
;
for
(
let
msgid
in
translation
.
getEntries
(
)
)
unusedEntries
[
msgid
]
=
true
;
return
Promise
.
all
(
this
.
__classes
.
map
(
async
classname
=>
{
if
(
!
classname
.
startsWith
(
library
.
getNamespace
(
)
)
)
{
return
;
}
let
dbClassInfo
=
await
Promisify
.
call
(
cb
=>
this
.
getClassInfo
(
classname
,
cb
)
)
;
if
(
!
dbClassInfo
.
translations
)
return
;
dbClassInfo
.
translations
.
forEach
(
function
(
src
)
{
var
entry
=
translation
.
getOrCreateEntry
(
src
.
msgid
)
;
delete
unusedEntries
[
src
.
msgid
]
;
if
(
src
.
msgid_plural
)
{
entry
.
msgid_plural
=
src
.
msgid_plural
;
}
if
(
!
entry
.
comments
)
{
entry
.
comments
=
{
}
;
}
entry
.
comments
.
extracted
=
src
.
comment
;
entry
.
comments
.
reference
=
{
}
;
let
ref
=
entry
.
comments
.
reference
;
const
fileName
=
classname
.
replace
(
/
\.
/
g
,
"/"
)
+
".js"
;
const
fnAddReference
=
lineNo
=>
{
let
arr
=
ref
[
fileName
]
;
if
(
!
arr
)
arr
=
ref
[
fileName
]
=
[
]
;
if
(
!
arr
.
includes
(
src
.
lineNo
)
)
{
arr
.
push
(
lineNo
)
;
}
}
;
if
(
qx
.
lang
.
Type
.
isArray
(
src
.
lineNo
)
)
{
src
.
lineNo
.
forEach
(
fnAddReference
)
;
}
else
{
fnAddReference
(
src
.
lineNo
)
;
}
}
)
;
}
)
)
.
then
(
(
)
=>
{
Object
.
keys
(
unusedEntries
)
.
forEach
(
msgid
=>
{
var
entry
=
translation
.
getEntry
(
msgid
)
;
if
(
entry
)
{
if
(
!
entry
.
comments
)
{
entry
.
comments
=
{
}
;
}
entry
.
comments
.
extracted
=
"NO LONGER USED"
;
entry
.
comments
.
reference
=
{
}
;
}
}
)
;
}
)
;
}
)
.
then
(
(
)
=>
translation
.
write
(
)
)
;
}
)
)
;
}

Updates all translations to include all msgids found in code
@param the library to update
@param locales
@param cb


function(name) {
      var t = this;
      for (var j = 0; j < t.__libraries.length; j++) {
        var library = t.__libraries[j];
        var info = library.getSymbolType(name);
        if (info) {
          return info;
        }
      }
      return null;
    }

function ( name ) { var t = this ; for ( var j = 0 ; j < t . __libraries . length ; j ++ ) { var library = t . __libraries [ j ] ; var info = library . getSymbolType ( name ) ; if ( info ) { return info ; } } return null ; }

Detects the symbol type, ie class, package, member, etc
@param name
@returns {{symbolType,name,clasName?}}

Detects the symbol type ie class package member etc


function(className) {
      var t = this;
      var info = this.__classFiles[className];
      if (info) {
        return info.library;
      }

      for (var j = 0; j < t.__libraries.length; j++) {
        var library = t.__libraries[j];
        info = library.getSymbolType(className);
        if (info && (info.symbolType == "class" || info.symbolType == "member")) {
          return library;
        }
      }

      return null;
    }

function ( className ) { var t = this ; var info = this . __classFiles [ className ] ; if ( info ) { return info . library ; } for ( var j = 0 ; j < t . __libraries . length ; j ++ ) { var library = t . __libraries [ j ] ; info = library . getSymbolType ( className ) ; if ( info && ( info . symbolType == "class" || info . symbolType == "member" ) ) { return library ; } } return null ; }

Returns the library for a given classname, supports private files
@param className
@returns {*}

Returns the library for a given classname supports private files


function(className) {
      var library = this.getLibraryFromClassname(className);
      if (!library) {
        return null;
      }
      var path = library.getRootDir() + "/" + library.getSourcePath() + "/" + className.replace(/\./g, "/") + ".js";
      return path;
    }

function ( className ) { var library = this . getLibraryFromClassname ( className ) ; if ( ! library ) { return null ; } var path = library . getRootDir ( ) + "/" + library . getSourcePath ( ) + "/" + className . replace ( / \. / g , "/" ) + ".js" ; return path ; }

Returns the classname
@param className
@returns {string}


function(key, value) {
      if (typeof key == "object") {
        var map = key;
        for (key in map) {
          this.__environmentChecks[key] = map[key];
        }
      } else if (value === undefined) {
        delete this.__environmentChecks[key];
      } else {
        this.__environmentChecks[key] = value;
      }
    }

function ( key , value ) { if ( typeof key == "object" ) { var map = key ; for ( key in map ) { this . __environmentChecks [ key ] = map [ key ] ; } } else if ( value === undefined ) { delete this . __environmentChecks [ key ] ; } else { this . __environmentChecks [ key ] = value ; } }

Sets an environment value as being checked for

@param key
@param value


function() {
      if (this.__qooxdooVersion) {
        return this.__qooxdooVersion;
      }
      if (!this.__qooxdooVersion) {
         let lib = this.findLibrary("qx");
         if (lib) {
          this.__qooxdooVersion = lib.getVersion();
         }
      }
      return this.__qooxdooVersion;
    }

function ( ) { if ( this . __qooxdooVersion ) { return this . __qooxdooVersion ; } if ( ! this . __qooxdooVersion ) { let lib = this . findLibrary ( "qx" ) ; if ( lib ) { this . __qooxdooVersion = lib . getVersion ( ) ; } } return this . __qooxdooVersion ; }

Returns the version of Qooxdoo
@returns {String}


function() {
      var m = this.__dbFilename.match(/(^.*)\/([^/]+)$/);
      var resDb;
      if (m && m.length == 3) {
        resDb = m[1] + "/resource-db.json";
      } else {
        resDb = "resource-db.json";
      }
      return resDb;
    }

function ( ) { var m = this . __dbFilename . match ( / (^.*)\/([^/]+)$ / ) ; var resDb ; if ( m && m . length == 3 ) { resDb = m [ 1 ] + "/resource-db.json" ; } else { resDb = "resource-db.json" ; } return resDb ; }

Returns the resource database filename
@returns {null}


function(comment) {
      var current = { name: "@description", body: "" };
      var cmds = [ current ];
      if (typeof comment == "string") {
        comment = comment.split("\n");
      }
      comment.forEach(function(line) {
        // Strip optional leading *
        line = line.trim();
        var m = line.match(/^\*\s?(.*)$/);
        if (m) {
          line = m[1];
        }
        line = line.trim();

        // Look for command at the begining of the line
        m = line.match(/^(\@[a-zA-Z0-9_]+)(.*)$/);
        if (!m) {
          if (current.body.length) {
            current.body += "\n";
          }
          current.body += line;
          return;
        }

        var name = m[1];
        var body = m[2];

        // Patch common command names
        if (name == "@returns") {
          name = "@return";
        }
        if (name == "@throw") {
          name = "@throws";
        }

        // store it
        current = { name: name, body: body };
        cmds.push(current);
      });
      var result = {};
      cmds.forEach(function(cmd) {
        // If the body is surrounded by parameters, remove them
        var m = cmd.body.match(/^\s*\(([\s\S]*)\)\s*$/);
        if (m) {
          cmd.body = m[1];
        }
        cmd.body = cmd.body.trim();
        if (result[cmd.name]) {
          result[cmd.name].push(cmd);
        } else {
          result[cmd.name] = [ cmd ];
        }
      });
      return result;
    }

Parses a JSDoc comment, returns an object where the keys are the name of the JSDoc (eg "@description")
and the value are an array of objects, one for each entry that was found.  The individual entries
consist of the name, the body of the JSDoc entry, and optional, key-specific parsed data (where supported)

Parses a JSDoc comment returns an object where the keys are the name of the JSDoc ( eg


function() {
      return path.join(process.cwd(), qx.tool.cli.ConfigSchemas.lockfile.filename);
    }

function ( ) { return path . join ( process . cwd ( ) , qx . tool . cli . ConfigSchemas . lockfile . filename ) ; }

Returns the absolute path to the contrib.json file
@return {String}

Returns the absolute path to the contrib . json file


async function() {
      let contrib_json_path = this.getContribFileName();
      if (!await fs.existsAsync(contrib_json_path)) {
        return {
          version: qx.tool.cli.ConfigSchemas.lockfile.version,
          libraries: []
        };
      }
      return qx.tool.compiler.utils.Json.loadJsonAsync(contrib_json_path);
    }

async function ( ) { let contrib_json_path = this . getContribFileName ( ) ; if ( ! await fs . existsAsync ( contrib_json_path ) ) { return { version : qx . tool . cli . ConfigSchemas . lockfile . version , libraries : [ ] } ; } return qx . tool . compiler . utils . Json . loadJsonAsync ( contrib_json_path ) ; }

Returns the library list from the contrib file
@return {Object}


async function(repo_name, library_name) {
      let library = (await this.getContribData()).libraries.find(lib => lib.repo_name === repo_name && lib.library_name === library_name);
      return library ? library.repo_tag : false;
    }

async function ( repo_name , library_name ) { let library = ( await this . getContribData ( ) ) . libraries . find ( lib => lib . repo_name === repo_name && lib . library_name === library_name ) ; return library ? library . repo_tag : false ; }

Returns the tag name of the given library in the given contrib repository, if installed.
Returns false if not installed.
@param {String} repo_name
@param {String} library_name
@return {String|false}

Returns the tag name of the given library in the given contrib repository if installed . Returns false if not installed .


async function(library_name) {
      return (await this.getContribData()).libraries.find(lib => lib.library_name === library_name);
    }

async function ( library_name ) { return ( await this . getContribData ( ) ) . libraries . find ( lib => lib . library_name === library_name ) ; }

Returns the data of the given library, if installed.
Returns false if not installed.
@param {String} library_name
@return {Object|false}

Returns the data of the given library if installed . Returns false if not installed .


function(readFromFile = false) {
      if (!readFromFile && this.__cache && typeof this.__cache == "object") {
        return this.__cache;
      }
      try {
        this.__cache = jsonlint.parse(fs.readFileSync(this.getCachePath(), "UTF-8"));
      } catch (e) {
        this.__cache = {
          repos : {
            list : [],
            data : {}
          },
          compat : {}
        };
      }
      return this.__cache;
    }

function ( readFromFile = false ) { if ( ! readFromFile && this . __cache && typeof this . __cache == "object" ) { return this . __cache ; } try { this . __cache = jsonlint . parse ( fs . readFileSync ( this . getCachePath ( ) , "UTF-8" ) ) ; } catch ( e ) { this . __cache = { repos : { list : [ ] , data : { } } , compat : { } } ; } return this . __cache ; }

Returns the cache object, retrieving it from a local file if necessary
@return {Object}

Returns the cache object retrieving it from a local file if necessary


async function() {
      await qx.tool.cli.Utils.makeParentDir(this.getCachePath());
      await fs.writeFileAsync(this.getCachePath(), JSON.stringify(this.__cache, null, 2), "UTF-8");
    }

async function ( ) { await qx . tool . cli . Utils . makeParentDir ( this . getCachePath ( ) ) ; await fs . writeFileAsync ( this . getCachePath ( ) , JSON . stringify ( this . __cache , null , 2 ) , "UTF-8" ) ; }

Saves the cache to a hidden local file
@return {void}


async function(path) {
      return fs.writeFileAsync(path, JSON.stringify(this.__cache, null, 2), "UTF-8")
        .catch(e => console.error(`Error exporting cache to ${path}:` + e.message));
    }

async function ( path ) { return fs . writeFileAsync ( path , JSON . stringify ( this . __cache , null , 2 ) , "UTF-8" ) . catch ( e => console . error ( ` ${ path } ` + e . message ) ) ; }

Exports the cache to an external file. Note that the structure of the cache
data can change any time. Do not build anything on it. You have been warned.
@param path {String}
@return {void}

Exports the cache to an external file . Note that the structure of the cache data can change any time . Do not build anything on it . You have been warned .


function expandMemberExpression(str) {
  var segs = str.split(".");
  var expr = types.memberExpression(types.identifier(segs[0]), types.identifier(segs[1]));
  for (var i = 2; i < segs.length; i++) {
    expr = types.memberExpression(expr, types.identifier(segs[i]));
  }
  return expr;
}

function expandMemberExpression ( str ) { var segs = str . split ( "." ) ; var expr = types . memberExpression ( types . identifier ( segs [ 0 ] ) , types . identifier ( segs [ 1 ] ) ) ; for ( var i = 2 ; i < segs . length ; i ++ ) { expr = types . memberExpression ( expr , types . identifier ( segs [ i ] ) ) ; } return expr ; }

Helper method that expands a dotted string into MemberExpression
@param str
@returns {*}


function(callback) {
      var t = this;
      var className = this.__className;
      t.__fatalCompileError = false;
      t.__numClassesDefined = 0;

      fs.readFile(this.getSourcePath(), {encoding: "utf-8"}, function(err, src) {
        if (err) {
          callback(err);
          return;
        }

        try {
          let options = t.__analyser.getBabelOptions() || {};
          options.modules = false;
          var config = {
              babelrc: false,
              sourceFileName : t.getSourcePath(),
              filename: t.getSourcePath(),
              sourceMaps: true,
              "presets": [
                [ require.resolve("@babel/preset-env"), options]
              ],
              plugins: [
                t._babelClassPlugin()
              ],
              parserOpts: { sourceType: "script" },
              passPerPreset: true
            };
          var result = babelCore.transform(src, config);
        } catch (ex) {
          if (ex._babel) {
            console.log(ex);
          }
          t.addMarker("compiler.syntaxError", ex.loc, ex.message, ex.codeFrame);
          t.__fatalCompileError = true;
          t._compileDbClassInfo();
          callback();
          return;
        }

        if (!t.__numClassesDefined) {
          t.addMarker("compiler.missingClassDef");
          t.__fatalCompileError = true;
          t._compileDbClassInfo();
          callback();
          return;
        }

        if (!t.__metaDefinitions[className]) {
          t.addMarker("compiler.wrongClassName", null, className, Object.keys(t.__metaDefinitions).join(", "));
          t._compileDbClassInfo();
        }

        var pos = className.lastIndexOf(".");
        var name = pos > -1 ? className.substring(pos + 1) : className;
        var outputPath = t.getOutputPath();
        util.mkParentPath(outputPath, function(err) {
          if (err) {
            callback(err);
            return;
          }
          fs.writeFile(outputPath, result.code + "\n\n//# sourceMappingURL=" + name + ".js.map?dt=" + (new Date().getTime()), {encoding: "utf-8"}, function(err) {
            if (err) {
              callback(err);
              return;
            }
            fs.writeFile(outputPath + ".map", JSON.stringify(result.map, null, 2), {encoding: "utf-8"}, function(err) {
              if (err) {
                callback(err);
                return;
              }
              t._waitForTaskQueueDrain(function() {
                callback();
              });
            });
          });
        });
      });
    }

function
(
callback
)
{
var
t
=
this
;
var
className
=
this
.
__className
;
t
.
__fatalCompileError
=
false
;
t
.
__numClassesDefined
=
0
;
fs
.
readFile
(
this
.
getSourcePath
(
)
,
{
encoding
:
"utf-8"
}
,
function
(
err
,
src
)
{
if
(
err
)
{
callback
(
err
)
;
return
;
}
try
{
let
options
=
t
.
__analyser
.
getBabelOptions
(
)
||
{
}
;
options
.
modules
=
false
;
var
config
=
{
babelrc
:
false
,
sourceFileName
:
t
.
getSourcePath
(
)
,
filename
:
t
.
getSourcePath
(
)
,
sourceMaps
:
true
,
"presets"
:
[
[
require
.
resolve
(
"@babel/preset-env"
)
,
options
]
]
,
plugins
:
[
t
.
_babelClassPlugin
(
)
]
,
parserOpts
:
{
sourceType
:
"script"
}
,
passPerPreset
:
true
}
;
var
result
=
babelCore
.
transform
(
src
,
config
)
;
}
catch
(
ex
)
{
if
(
ex
.
_babel
)
{
console
.
log
(
ex
)
;
}
t
.
addMarker
(
"compiler.syntaxError"
,
ex
.
loc
,
ex
.
message
,
ex
.
codeFrame
)
;
t
.
__fatalCompileError
=
true
;
t
.
_compileDbClassInfo
(
)
;
callback
(
)
;
return
;
}
if
(
!
t
.
__numClassesDefined
)
{
t
.
addMarker
(
"compiler.missingClassDef"
)
;
t
.
__fatalCompileError
=
true
;
t
.
_compileDbClassInfo
(
)
;
callback
(
)
;
return
;
}
if
(
!
t
.
__metaDefinitions
[
className
]
)
{
t
.
addMarker
(
"compiler.wrongClassName"
,
,
className
,
Object
.
keys
(
t
.
__metaDefinitions
)
.
join
(
", "
)
)
;
t
.
_compileDbClassInfo
(
)
;
}
var
pos
=
className
.
lastIndexOf
(
"."
)
;
var
name
=
pos
>
1
className
.
substring
(
pos
+
1
)
:
className
;
var
outputPath
=
t
.
getOutputPath
(
)
;
util
.
mkParentPath
(
outputPath
,
function
(
err
)
{
if
(
err
)
{
callback
(
err
)
;
return
;
}
fs
.
writeFile
(
outputPath
,
result
.
code
+
"\n\n//# sourceMappingURL="
+
name
+
".js.map?dt="
+
(
new
Date
(
)
.
getTime
(
)
)
,
{
encoding
:
"utf-8"
}
,
function
(
err
)
{
if
(
err
)
{
callback
(
err
)
;
return
;
}
fs
.
writeFile
(
outputPath
+
".map"
,
JSON
.
stringify
(
result
.
map
,
,
2
)
,
{
encoding
:
"utf-8"
}
,
function
(
err
)
{
if
(
err
)
{
callback
(
err
)
;
return
;
}
t
.
_waitForTaskQueueDrain
(
function
(
)
{
callback
(
)
;
}
)
;
}
)
;
}
)
;
}
)
;
}
)
;
}

Loads the source, transpiles and analyses the code, storing the result in outputPath

@param callback
{Function} callback for when the load is completed

Loads the source transpiles and analyses the code storing the result in outputPath


function(className) {
      var pos = className.lastIndexOf(".");
      var meta = {
          className: className,
          packageName: pos > -1 ? className.substring(0, pos) : null,
          name: pos > -1 ? className.substring(pos + 1) : className,
          superClass: null,
          interfaces: [],
          mixins: [],
          functionName: null
      };
      this.__metaStack.push(meta);
      this.__classMeta = meta;
      this.__metaDefinitions[className] = meta;
      this.__numClassesDefined++;
    }

function ( className ) { var pos = className . lastIndexOf ( "." ) ; var meta = { className : className , packageName : pos > - 1 ? className . substring ( 0 , pos ) : null , name : pos > - 1 ? className . substring ( pos + 1 ) : className , superClass : null , interfaces : [ ] , mixins : [ ] , functionName : null } ; this . __metaStack . push ( meta ) ; this . __classMeta = meta ; this . __metaDefinitions [ className ] = meta ; this . __numClassesDefined ++ ; }

Pushes new meta data onto the stack - each meta represents a class being defined,
we operate a stack so that we can handle inner classes

@param className {String} name of the class being defined

Pushes new meta data onto the stack - each meta represents a class being defined we operate a stack so that we can handle inner classes


function(className) {
      if (!this.__metaStack.length) {
        throw new Error("No __metaStack entries to pop");
      }
      let meta = this.__metaStack[this.__metaStack.length - 1];
      if (className && meta.className != className) {
        throw new Error("Wrong __metaStack entries to pop, expected " + className + " found " + meta.className);
      }
      this.__metaStack.pop();
      meta = this.__metaStack[this.__metaStack.length - 1] || null;
      this.__classMeta = meta;
    }

Pops the current meta off the stack, optionally checking that the classname is correct

Pops the current meta off the stack optionally checking that the classname is correct


function(functionName, node, isClassMember) {
      this.__scope = {
        functionName: functionName,
        parent: this.__scope,
        vars: {},
        unresolved: {},
        isClassMember: !!isClassMember
      };
    }

function ( functionName , node , isClassMember ) { this . __scope = { functionName : functionName , parent : this . __scope , vars : { } , unresolved : { } , isClassMember : ! ! isClassMember } ; }


function(node) {
      var old = this.__scope;
      var scope = this.__scope = this.__scope.parent;
      var unresolved = scope.unresolved;

      for (var name in old.unresolved) {
        var entry = unresolved[name];
        if (!entry) {
          entry = unresolved[name] = {
            name: name,
            locations: []
          };
        }
        entry.locations.push.apply(entry.locations, old.unresolved[name].locations);
      }
    }

function ( node ) { var old = this . __scope ; var scope = this . __scope = this . __scope . parent ; var unresolved = scope . unresolved ; for ( var name in old . unresolved ) { var entry = unresolved [ name ] ; if ( ! entry ) { entry = unresolved [ name ] = { name : name , locations : [ ] } ; } entry . locations . push . apply ( entry . locations , old . unresolved [ name ] . locations ) ; } }