stickshift.js
function takeToken(cm, pos, precise, asArray) {
    function getObj(copy) {
      return {start: stream.start, end: stream.pos,
              string: stream.current(),
              type: style || null,
              state: copy ? copyState(doc.mode, state) : state};
    }

    var doc = cm.doc, mode = doc.mode, style;
    pos = clipPos(doc, pos);
    var line = getLine(doc, pos.line), state = getStateBefore(cm, pos.line, precise);
    var stream = new StringStream(line.text, cm.options.tabSize), tokens;
    if (asArray) tokens = [];
    while ((asArray || stream.pos < pos.ch) && !stream.eol()) {
      stream.start = stream.pos;
      style = readToken(mode, stream, state);
      if (asArray) tokens.push(getObj(true));
    }
    return asArray ? tokens : getObj();
  }
function takeToken(cm, pos, precise, asArray) {
    function getObj(copy) {
      return {start: stream.start, end: stream.pos,
              string: stream.current(),
              type: style || null,
              state: copy ? copyState(doc.mode, state) : state};
    }

    var doc = cm.doc, mode = doc.mode, style;
    pos = clipPos(doc, pos);
    var line = getLine(doc, pos.line), state = getStateBefore(cm, pos.line, precise);
    var stream = new StringStream(line.text, cm.options.tabSize), tokens;
    if (asArray) tokens = [];
    while ((asArray || stream.pos < pos.ch) && !stream.eol()) {
      stream.start = stream.pos;
      style = readToken(mode, stream, state);
      if (asArray) tokens.push(getObj(true));
    }
    return asArray ? tokens : getObj();
  }
function takeToken ( cm , pos , precise , asArray ) { function getObj ( copy ) { return { start : stream . start , end : stream . pos , string : stream . current ( ) , type : style || null , state : copy ? copyState ( doc . mode , state ) : state } ; } var doc = cm . doc , mode = doc . mode , style ; pos = clipPos ( doc , pos ) ; var line = getLine ( doc , pos . line ) , state = getStateBefore ( cm , pos . line , precise ) ; var stream = new StringStream ( line . text , cm . options . tabSize ) , tokens ; if ( asArray ) tokens = [ ] ; while ( ( asArray || stream . pos < pos . ch ) && ! stream . eol ( ) ) { stream . start = stream . pos ; style = readToken ( mode , stream , state ) ; if ( asArray ) tokens . push ( getObj ( true ) ) ; } return asArray ? tokens : getObj ( ) ; }
Utility for getTokenAt and getLineTokens
Utility for getTokenAt and getLineTokens