lib/lexer.js
function reader(str) {
  var matched;

  utils.some(rules, function (rule) {
    return utils.some(rule.regex, function (regex) {
      var match = str.match(regex),
        normalized;

      if (!match) {
        return;
      }

      normalized = match[rule.idx || 0].replace(/\s*$/, '');
      normalized = (rule.hasOwnProperty('replace') && rule.replace.hasOwnProperty(normalized)) ? rule.replace[normalized] : normalized;

      matched = {
        match: normalized,
        type: rule.type,
        length: match[0].length
      };
      return true;
    });
  });

  if (!matched) {
    matched = {
      match: str,
      type: TYPES.UNKNOWN,
      length: str.length
    };
  }

  return matched;
}
function reader(str) {
  var matched;

  utils.some(rules, function (rule) {
    return utils.some(rule.regex, function (regex) {
      var match = str.match(regex),
        normalized;

      if (!match) {
        return;
      }

      normalized = match[rule.idx || 0].replace(/\s*$/, '');
      normalized = (rule.hasOwnProperty('replace') && rule.replace.hasOwnProperty(normalized)) ? rule.replace[normalized] : normalized;

      matched = {
        match: normalized,
        type: rule.type,
        length: match[0].length
      };
      return true;
    });
  });

  if (!matched) {
    matched = {
      match: str,
      type: TYPES.UNKNOWN,
      length: str.length
    };
  }

  return matched;
}
function reader ( str ) { var matched ; utils . some ( rules , function ( rule ) { return utils . some ( rule . regex , function ( regex ) { var match = str . match ( regex ) , normalized ; if ( ! match ) { return ; } normalized = match [ rule . idx || 0 ] . replace ( / \s*$ / , '' ) ; normalized = ( rule . hasOwnProperty ( 'replace' ) && rule . replace . hasOwnProperty ( normalized ) ) ? rule . replace [ normalized ] : normalized ; matched = { match : normalized , type : rule . type , length : match [ 0 ] . length } ; return true ; } ) ; } ) ; if ( ! matched ) { matched = { match : str , type : TYPES . UNKNOWN , length : str . length } ; } return matched ; }
Return the token type object for a single chunk of a string.
@param  {string} str String chunk.
@return {LexerToken}     Defined type, potentially stripped or replaced with more suitable content.
@private
Return the token type object for a single chunk of a string .