quantum-core/lib/parse.js
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
function tokenize ( str ) { let pos = 0 let row = 1 let col = 1 let newline = true let state = CONTENT let start = 0 let consumingSameLineContent = false let escapedInlineCounter = 0 let escapedParamCounter = 0 let consumingEscapedParams = false let consumingUnparsed = false // true when in an @@ block let lastInlineContentEndPos = void ( 0 ) let unparsedIndentStart = void ( 0 ) const indent = [ 0 ] const tokens = [ ] function emit ( type , value ) { if ( arguments . length > 1 ) { tokens . push ( { type : type , value : value } ) } else { tokens . push ( { type : type } ) } } function consume ( type , next ) { emit ( type , str . substring ( start , pos ) ) state = next start = pos + 1 } function consumeIfNonEmpty ( type , next , escaped ) { let v = str . substring ( start , pos ) if ( v . length > 0 ) { if ( escaped ) { const l = escaped . length for ( let i = 0 ; i < l ; i ++ ) { const es = escaped [ i ] v = v . replace ( es , es [ 1 ] ) } } emit ( type , v ) } state = next start = pos + 1 } function shuffleIfNext ( character ) { if ( str [ pos + 1 ] === character ) { pos ++ start ++ return true } else { return false } } function err ( msg ) { let start = str . lastIndexOf ( '\n' , pos - 1 ) start = str . lastIndexOf ( '\n' , start - 1 ) start = str . lastIndexOf ( '\n' , start - 1 ) const nextNewline = str . indexOf ( '\n' , pos ) const errorLineEnd = nextNewline === - 1 ? pos : nextNewline let end = str . indexOf ( '\n' , errorLineEnd + 1 ) if ( end === - 1 ) { end = str . length - 1 } else { end = str . indexOf ( '\n' , end + 1 ) if ( end === - 1 ) { end = str . length - 1 } else { end = str . indexOf ( '\n' , end + 1 ) if ( end === - 1 ) { end = str . length - 1 } } } const indentSpaces = ( new Array ( col ) ) . join ( ' ' ) const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg const context = str . substring ( start , errorLineEnd ) + '\n' + indentSpaces + '^^^^^^^^\n' + str . substring ( errorLineEnd + 1 , end ) throw new ParseError ( message + '\n' + context , context , str , row , col , msg , pos ) } while ( pos < str . length ) { // indentation and comment handling. if ( newline ) { while ( newline && pos < str . length ) { let ind = 0 while ( str [ pos + ind ] === ' ' && pos + ind < str . length ) { ind ++ } // if it was an empty content line, then add it to the list of empty // content to emit on the next non empty line if ( str [ pos + ind ] === '\n' ) { pos += ind + 1 row ++ emit ( 'EMPTY_CONTENT' , str . substring ( start , pos - 1 ) ) } else if ( str [ pos + ind ] !== '#' || consumingUnparsed ) { if ( ind > indent [ indent . length - 1 ] ) { emit ( 'INDENT' , ind - indent [ indent . length - 1 ] ) indent . push ( ind ) } else if ( ind < indent [ indent . length - 1 ] ) { while ( indent [ indent . length - 1 ] > ind ) { const prev = indent . pop ( ) emit ( 'DEDENT' , prev - indent [ indent . length - 1 ] ) if ( consumingUnparsed && unparsedIndentStart >= indent [ indent . length - 1 ] ) { consumingUnparsed = false } } if ( indent . length > 0 && ( indent [ indent . length - 1 ] !== ind ) ) { err ( 'indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent [ indent . length - 1 ] + ' spaces was expected' ) } } pos += ind if ( str [ pos ] === '\\' && str [ pos + 1 ] === '#' ) { pos ++ } newline = false } else { while ( str [ pos ] !== '\n' && pos < str . length ) { pos ++ } emit ( 'COMMENT' , str . substring ( start + ind + 1 , pos ) ) pos ++ row ++ } start = pos } } if ( pos >= str . length ) { return tokens } const s = str [ pos ] if ( state === TYPE ) { if ( s === ' ' ) { consume ( 'TYPE' , PARAMS ) while ( shuffleIfNext ( ' ' ) && pos < str . length ) { } } else if ( s === ':' ) { consume ( 'TYPE' , CONTENT ) shuffleIfNext ( ' ' ) consumingSameLineContent = true emit ( 'START_SAME_LINE_CONTENT' ) } else if ( s === '[' ) { consume ( 'TYPE' , INLINE_CONTENT ) emit ( 'START_INLINE_CONTENT' ) } else if ( s === '(' ) { consume ( 'TYPE' , INLINE_PARAMS ) } else if ( s === '\n' ) { consume ( 'TYPE' , CONTENT ) if ( consumingSameLineContent ) { emit ( 'END_SAME_LINE_CONTENT' ) consumingSameLineContent = false } } } else if ( state === PARAMS ) { if ( s === '[' ) { escapedParamCounter ++ consumingEscapedParams = true } else if ( consumingEscapedParams && ( s === ']' ) ) { escapedParamCounter -- if ( escapedParamCounter === 0 ) { consumingEscapedParams = false } } else if ( s === ':' && ! consumingEscapedParams ) { consume ( 'PARAMS' , CONTENT ) shuffleIfNext ( ' ' ) shuffleIfNext ( ']' ) consumingSameLineContent = true emit ( 'START_SAME_LINE_CONTENT' ) } else if ( s === '\n' ) { escapedParamCounter = 0 consumingEscapedParams = false consume ( 'PARAMS' , CONTENT ) if ( consumingSameLineContent ) { emit ( 'END_SAME_LINE_CONTENT' ) consumingSameLineContent = false } } } else if ( state === CONTENT ) { if ( s === '@' && ! consumingUnparsed ) { consumeIfNonEmpty ( 'CONTENT' , TYPE ) if ( str [ pos + 1 ] === '@' ) { consumingUnparsed = true unparsedIndentStart = indent [ indent . length - 1 ] pos ++ start ++ } } else if ( s === '\n' ) { if ( consumingSameLineContent ) { if ( lastInlineContentEndPos !== pos - 1 ) { consumeIfNonEmpty ( 'CONTENT' , CONTENT ) } emit ( 'END_SAME_LINE_CONTENT' ) consumingSameLineContent = false consumingUnparsed = false } else { if ( lastInlineContentEndPos !== pos - 1 ) { consumeIfNonEmpty ( 'CONTENT' , CONTENT ) } } } } else if ( state === INLINE_PARAMS ) { if ( s === ')' ) { if ( str [ pos + 1 ] === '[' ) { consume ( 'PARAMS' , INLINE_CONTENT ) emit ( 'START_INLINE_CONTENT' ) start ++ pos ++ } else { consume ( 'PARAMS' , CONTENT ) } } } else { // if (state === INLINE_CONTENT) { if ( s === '\\' && ( str [ pos + 1 ] === '[' || str [ pos + 1 ] === ']' ) ) { pos ++ } else if ( s === '[' ) { escapedInlineCounter ++ } else if ( s === ']' ) { if ( escapedInlineCounter === 0 ) { consumeIfNonEmpty ( 'CONTENT' , CONTENT , [ '\\]' , '\\[' ] ) emit ( 'END_INLINE_CONTENT' ) lastInlineContentEndPos = pos + 1 } else { escapedInlineCounter -- } } else if ( s === '\n' ) { consume ( 'CONTENT' , INLINE_CONTENT ) } } if ( s === '\n' ) { col = 0 row ++ newline = true } else { col ++ } pos ++ } if ( state === TYPE ) { emit ( 'TYPE' , str . substring ( start , pos ) ) } else if ( state === PARAMS ) { emit ( 'PARAMS' , str . substring ( start , pos ) ) } else if ( state === INLINE_PARAMS ) { err ( 'missing closing ) bracket?' ) } else if ( state === INLINE_CONTENT ) { err ( 'missing closing ] bracket?' ) } else { // if (state === CONTENT) { if ( start < pos ) { emit ( 'CONTENT' , str . substring ( start , pos ) ) } } return tokens }
turns the input into an array of tokens
turns the input into an array of tokens