lexer.js
function match_in_lextabs(lexer, options) {
  options = options || {};
  var token;
  var replaced = {};

  // for each dialect's lextab...
  for(var d = 0; !token && d < lexer.dialects.length; d++) {
    var dialect = lexer.dialects[d];
    if(!dialect.lextab) {
      continue; // no lextab for the dialect so move on
    }

    // check each entry in order...
// NOTE I AM IGNORING THE 'PRIORITY' PROPERTY RIGHT NOW - MAYBE I CAN DELETE THEM!?
    debug('matching in ' + dialect.name + ' lextable');
    for(var l = 0; !token && l < dialect.lextab.length; l++) {
      var entry = dialect.lextab[l];

      // we don't match tokens against "replaced" categories
      if(!replaced[entry.category] &&
        // and if they've specified categories to include...
        (!options.include ||
          // only consider those
          (options.include.contains(entry.category) ||
          (options.include.contains("default") && entry.default))) &&
        // or if they've specified categories to omit...
        (!options.omit ||
          // make sure we skip over those
          ((!options.omit.contains(entry.category) &&
          !(options.omit.contains("default") && entry.default)))))
      {
        // most functions "match" (with a regex)
        // but they can also just "read" (with a function)
        if(typeof entry.read !== 'function') {
          // are we on a token matching this entry's pattern?
          //trace('matching ' + entry.category + ' pattern');
          var matchedText = lexer.on(entry.match);
          if(matchedText) {
            trace('matched ' + entry.category + ' pattern');
            // advance the current position
            lexer.next_char(matchedText.length);
            // create and return the token object (including line/col info)
            token = lexer.create_token(matchedText, entry.category);
          }
        }
        else {
          // note we use a "read" function for our default entry
          // used when nothing else matches.  Such entries can
          // still set a token category, but they should set
          // "default: true" (so we know to consider them last).
          trace('invoking ' + entry.category + ' read function');
          token = entry.read(lexer);
          if(token) {
            trace('read from ' + entry.category + ' read function');
            token.category = entry.category;
          }
        }
      }

      if(entry.replace) {
        // remember that this category has been replaced
        replaced[entry.category] = true;
      }
    }
  }

  return token;
}
function match_in_lextabs(lexer, options) {
  options = options || {};
  var token;
  var replaced = {};

  // for each dialect's lextab...
  for(var d = 0; !token && d < lexer.dialects.length; d++) {
    var dialect = lexer.dialects[d];
    if(!dialect.lextab) {
      continue; // no lextab for the dialect so move on
    }

    // check each entry in order...
// NOTE I AM IGNORING THE 'PRIORITY' PROPERTY RIGHT NOW - MAYBE I CAN DELETE THEM!?
    debug('matching in ' + dialect.name + ' lextable');
    for(var l = 0; !token && l < dialect.lextab.length; l++) {
      var entry = dialect.lextab[l];

      // we don't match tokens against "replaced" categories
      if(!replaced[entry.category] &&
        // and if they've specified categories to include...
        (!options.include ||
          // only consider those
          (options.include.contains(entry.category) ||
          (options.include.contains("default") && entry.default))) &&
        // or if they've specified categories to omit...
        (!options.omit ||
          // make sure we skip over those
          ((!options.omit.contains(entry.category) &&
          !(options.omit.contains("default") && entry.default)))))
      {
        // most functions "match" (with a regex)
        // but they can also just "read" (with a function)
        if(typeof entry.read !== 'function') {
          // are we on a token matching this entry's pattern?
          //trace('matching ' + entry.category + ' pattern');
          var matchedText = lexer.on(entry.match);
          if(matchedText) {
            trace('matched ' + entry.category + ' pattern');
            // advance the current position
            lexer.next_char(matchedText.length);
            // create and return the token object (including line/col info)
            token = lexer.create_token(matchedText, entry.category);
          }
        }
        else {
          // note we use a "read" function for our default entry
          // used when nothing else matches.  Such entries can
          // still set a token category, but they should set
          // "default: true" (so we know to consider them last).
          trace('invoking ' + entry.category + ' read function');
          token = entry.read(lexer);
          if(token) {
            trace('read from ' + entry.category + ' read function');
            token.category = entry.category;
          }
        }
      }

      if(entry.replace) {
        // remember that this category has been replaced
        replaced[entry.category] = true;
      }
    }
  }

  return token;
}
function match_in_lextabs ( lexer , options ) { options = options || { } ; var token ; var replaced = { } ; // for each dialect's lextab... for ( var d = 0 ; ! token && d < lexer . dialects . length ; d ++ ) { var dialect = lexer . dialects [ d ] ; if ( ! dialect . lextab ) { continue ; // no lextab for the dialect so move on } // check each entry in order... // NOTE I AM IGNORING THE 'PRIORITY' PROPERTY RIGHT NOW - MAYBE I CAN DELETE THEM!? debug ( 'matching in ' + dialect . name + ' lextable' ) ; for ( var l = 0 ; ! token && l < dialect . lextab . length ; l ++ ) { var entry = dialect . lextab [ l ] ; // we don't match tokens against "replaced" categories if ( ! replaced [ entry . category ] && // and if they've specified categories to include... ( ! options . include || // only consider those ( options . include . contains ( entry . category ) || ( options . include . contains ( "default" ) && entry . default ) ) ) && // or if they've specified categories to omit... ( ! options . omit || // make sure we skip over those ( ( ! options . omit . contains ( entry . category ) && ! ( options . omit . contains ( "default" ) && entry . default ) ) ) ) ) { // most functions "match" (with a regex) // but they can also just "read" (with a function) if ( typeof entry . read !== 'function' ) { // are we on a token matching this entry's pattern? //trace('matching ' + entry.category + ' pattern'); var matchedText = lexer . on ( entry . match ) ; if ( matchedText ) { trace ( 'matched ' + entry . category + ' pattern' ) ; // advance the current position lexer . next_char ( matchedText . length ) ; // create and return the token object (including line/col info) token = lexer . create_token ( matchedText , entry . category ) ; } } else { // note we use a "read" function for our default entry // used when nothing else matches.  Such entries can // still set a token category, but they should set // "default: true" (so we know to consider them last). trace ( 'invoking ' + entry . category + ' read function' ) ; token = entry . read ( lexer ) ; if ( token ) { trace ( 'read from ' + entry . category + ' read function' ) ; token . category = entry . category ; } } } if ( entry . replace ) { // remember that this category has been replaced replaced [ entry . category ] = true ; } } } return token ; }
Helper function returns a token for the source text
under the current position based on the nearest match
in the active dialect's lextabs.

You can optionally omit specified token categories from consideration
("options.omit"), or include only specified token categories for
consideration ("options.include").

You can also use the word "default" in options.omit/options.include
to refer to default lextab entries indicated with "default: true"
Helper function returns a token for the source text under the current position based on the nearest match in the active dialect s lextabs .