lib/schemas.js
function Tokenizer(str) {
  this._str = str;
  this._pos = 0;
  this._queue = new BoundedQueue(3); // Bounded queue of last emitted tokens.
  this._token = undefined; // Current token.
  this._doc = undefined; // Javadoc.
}
function Tokenizer(str) {
  this._str = str;
  this._pos = 0;
  this._queue = new BoundedQueue(3); // Bounded queue of last emitted tokens.
  this._token = undefined; // Current token.
  this._doc = undefined; // Javadoc.
}
function Tokenizer ( str ) { this . _str = str ; this . _pos = 0 ; this . _queue = new BoundedQueue ( 3 ) ; // Bounded queue of last emitted tokens. this . _token = undefined ; // Current token. this . _doc = undefined ; // Javadoc. }
Helpers. 
Simple class to split an input string into tokens.

There are different types of tokens, characterized by their `id`:

+ `number` numbers.
+ `name` references.
+ `string` double-quoted.
+ `operator`, anything else, always single character.
+ `json`, special, must be asked for (the tokenizer doesn't have enough
context to predict these).

This tokenizer also handles Javadoc extraction, via the `addJavadoc` method.
Helpers . Simple class to split an input string into tokens .