lib/compiler/tokenizer.js
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
function tokenizer ( input ) { const topLevelTokens = getTopLevelTokens ( ) ; const length = input . length ; const output = [ ] ; let cursor = 0 ; let lastBreak = 0 ; while ( cursor < length ) { const slice = input . slice ( cursor ) ; const found = matchPattern ( topLevelTokens , slice , false ) ; if ( found && input [ cursor - 1 ] === '\\' ) { const text = input . slice ( lastBreak , cursor - 1 ) ; if ( text ) { output . push ( new Text ( text ) ) ; } const escapedText = found [ 1 ] [ 0 ] ; output . push ( new Text ( escapedText ) ) ; cursor += escapedText . length ; lastBreak = cursor ; } else if ( found ) { const [ Tok , matches ] = found ; const text = input . slice ( lastBreak , cursor ) ; if ( text ) { output . push ( new Text ( text ) ) ; } output . push ( new Tok ( ... matches ) ) ; cursor += matches [ 0 ] . length ; lastBreak = cursor ; } else { cursor += 1 ; } } const text = input . slice ( lastBreak , cursor ) ; if ( text ) { output . push ( new Text ( text ) ) ; } // if there are more closes than opens // intelligently remove extra ones return removeExtraCloses ( output ) ; }
Generate an array of tokens describing the template
@param {string} input
@return {Token[]}
Generate an array of tokens describing the template