lexer.js
function next_lextab_token(lexer) {

  var token;

  if(lexer.eos()) {
    return undefined;
  }

  // skip leading whitespace or comments...
  lexer.skip_filler();

  var previouslyPeeked = lexer.getPeekedToken();
  if(previouslyPeeked) {
    return previouslyPeeked;
  }

  lexer.mark_token_start();
  trace(lexer.message_src_loc("", lexer, {file:false}));

  // try and match all token categories except punctuation (and
  // the default handler).  Note that single character punctuation
  // should take a back seat to longer symbols e.g. '...' should
  // match before '.'.
  token = match_in_lextabs(lexer, {omit: ['punctuation','default']});
  if(!token) {
    // okay now try again matching punctuation characters.
    token = match_in_lextabs(lexer, {include: ['punctuation']});
  }
  if(!token) {
    // ok go ahead and try any default handler(s)
    token = match_in_lextabs(lexer, {include: ['default']});
  }

  // we expect they will use an explicit default lextab entry, but JIC:
  if(!token) {
    trace('their lextab has no default handler - defaulting to next word');
    token = lexer.next_word_token();
    if(token) {
      token.category = 'symbol';
    }
  }

  return token;
}
function next_lextab_token(lexer) {

  var token;

  if(lexer.eos()) {
    return undefined;
  }

  // skip leading whitespace or comments...
  lexer.skip_filler();

  var previouslyPeeked = lexer.getPeekedToken();
  if(previouslyPeeked) {
    return previouslyPeeked;
  }

  lexer.mark_token_start();
  trace(lexer.message_src_loc("", lexer, {file:false}));

  // try and match all token categories except punctuation (and
  // the default handler).  Note that single character punctuation
  // should take a back seat to longer symbols e.g. '...' should
  // match before '.'.
  token = match_in_lextabs(lexer, {omit: ['punctuation','default']});
  if(!token) {
    // okay now try again matching punctuation characters.
    token = match_in_lextabs(lexer, {include: ['punctuation']});
  }
  if(!token) {
    // ok go ahead and try any default handler(s)
    token = match_in_lextabs(lexer, {include: ['default']});
  }

  // we expect they will use an explicit default lextab entry, but JIC:
  if(!token) {
    trace('their lextab has no default handler - defaulting to next word');
    token = lexer.next_word_token();
    if(token) {
      token.category = 'symbol';
    }
  }

  return token;
}
function next_lextab_token ( lexer ) { var token ; if ( lexer . eos ( ) ) { return undefined ; } // skip leading whitespace or comments... lexer . skip_filler ( ) ; var previouslyPeeked = lexer . getPeekedToken ( ) ; if ( previouslyPeeked ) { return previouslyPeeked ; } lexer . mark_token_start ( ) ; trace ( lexer . message_src_loc ( "" , lexer , { file : false } ) ) ; // try and match all token categories except punctuation (and // the default handler).  Note that single character punctuation // should take a back seat to longer symbols e.g. '...' should // match before '.'. token = match_in_lextabs ( lexer , { omit : [ 'punctuation' , 'default' ] } ) ; if ( ! token ) { // okay now try again matching punctuation characters. token = match_in_lextabs ( lexer , { include : [ 'punctuation' ] } ) ; } if ( ! token ) { // ok go ahead and try any default handler(s) token = match_in_lextabs ( lexer , { include : [ 'default' ] } ) ; } // we expect they will use an explicit default lextab entry, but JIC: if ( ! token ) { trace ( 'their lextab has no default handler - defaulting to next word' ) ; token = lexer . next_word_token ( ) ; if ( token ) { token . category = 'symbol' ; } } return token ; }
Get the next token under the current source position according
to the lextab entries of the current dialects.
Get the next token under the current source position according to the lextab entries of the current dialects .