src/scss/tokenizer.js
function getTokens(css) {
    // Parse string, character by character:
    for (pos = 0; pos < css.length; col++, pos++) {
      c = css.charAt(pos);
      cn = css.charAt(pos + 1);

      // If we meet `/*`, it's a start of a multiline comment.
      // Parse following characters as a multiline comment:
      if (c === '/' && cn === '*') {
        parseMLComment(css);
      }

      // If we meet `//` and it is not a part of url:
      else if (!urlMode && c === '/' && cn === '/') {
        // If we're currently inside a block, treat `//` as a start
        // of identifier. Else treat `//` as a start of a single-line
        // comment:
        parseSLComment(css);
      }

      // If current character is a double or single quote, it's a start
      // of a string:
      else if (c === '"' || c === "'") {
        parseString(css, c);
      }

      // If current character is a space:
      else if (c === ' ') {
        parseSpaces(css);
      }

      // If current character is `=`, it must be combined with next `=`
      else if (c === '=' && cn === '=') {
        parseEquality(css);
      }

      // If we meet `!=`, this must be inequality
      else if (c === '!' && cn === '=') {
        parseInequality(css);
      }

      // If current character is a punctuation mark:
      else if (c in Punctuation) {
        // Check for CRLF here or just LF
        if (c === '\r' && cn === '\n' || c === '\n') {
          // If \r we know the next character is \n due to statement above
          // so we push a CRLF token type to the token list and importantly
          // skip the next character so as not to double count newlines or
          // columns etc
          if (c === '\r') {
            pushToken(TokenType.Newline, '\r\n', col);
            pos++; // If CRLF skip the next character and push crlf token
          } else if (c === '\n') {
            // If just a LF newline and not part of CRLF newline we can just
            // push punctuation as usual
            pushToken(Punctuation[c], c, col);
          }

          ln++; // Go to next line
          col = 0; // Reset the column count
        } else if (c !== '\r' && c !== '\n') {
          // Handle all other punctuation and add to list of tokens
          pushToken(Punctuation[c], c, col);
        }// Go to next line
        if (c === ')') urlMode = false; // Exit url mode
        else if (c === '\t' && tabSize > 1) col += (tabSize - 1);
      }

      // If current character is a decimal digit:
      else if (isDecimalDigit(c)) {
        parseDecimalNumber(css);
      }

      // If current character is anything else:
      else {
        parseIdentifier(css);
      }
    }

    return tokens;
  }
function getTokens(css) {
    // Parse string, character by character:
    for (pos = 0; pos < css.length; col++, pos++) {
      c = css.charAt(pos);
      cn = css.charAt(pos + 1);

      // If we meet `/*`, it's a start of a multiline comment.
      // Parse following characters as a multiline comment:
      if (c === '/' && cn === '*') {
        parseMLComment(css);
      }

      // If we meet `//` and it is not a part of url:
      else if (!urlMode && c === '/' && cn === '/') {
        // If we're currently inside a block, treat `//` as a start
        // of identifier. Else treat `//` as a start of a single-line
        // comment:
        parseSLComment(css);
      }

      // If current character is a double or single quote, it's a start
      // of a string:
      else if (c === '"' || c === "'") {
        parseString(css, c);
      }

      // If current character is a space:
      else if (c === ' ') {
        parseSpaces(css);
      }

      // If current character is `=`, it must be combined with next `=`
      else if (c === '=' && cn === '=') {
        parseEquality(css);
      }

      // If we meet `!=`, this must be inequality
      else if (c === '!' && cn === '=') {
        parseInequality(css);
      }

      // If current character is a punctuation mark:
      else if (c in Punctuation) {
        // Check for CRLF here or just LF
        if (c === '\r' && cn === '\n' || c === '\n') {
          // If \r we know the next character is \n due to statement above
          // so we push a CRLF token type to the token list and importantly
          // skip the next character so as not to double count newlines or
          // columns etc
          if (c === '\r') {
            pushToken(TokenType.Newline, '\r\n', col);
            pos++; // If CRLF skip the next character and push crlf token
          } else if (c === '\n') {
            // If just a LF newline and not part of CRLF newline we can just
            // push punctuation as usual
            pushToken(Punctuation[c], c, col);
          }

          ln++; // Go to next line
          col = 0; // Reset the column count
        } else if (c !== '\r' && c !== '\n') {
          // Handle all other punctuation and add to list of tokens
          pushToken(Punctuation[c], c, col);
        }// Go to next line
        if (c === ')') urlMode = false; // Exit url mode
        else if (c === '\t' && tabSize > 1) col += (tabSize - 1);
      }

      // If current character is a decimal digit:
      else if (isDecimalDigit(c)) {
        parseDecimalNumber(css);
      }

      // If current character is anything else:
      else {
        parseIdentifier(css);
      }
    }

    return tokens;
  }
function getTokens ( css ) { // Parse string, character by character: for ( pos = 0 ; pos < css . length ; col ++ , pos ++ ) { c = css . charAt ( pos ) ; cn = css . charAt ( pos + 1 ) ; // If we meet `/*`, it's a start of a multiline comment. // Parse following characters as a multiline comment: if ( c === '/' && cn === '*' ) { parseMLComment ( css ) ; } // If we meet `//` and it is not a part of url: else if ( ! urlMode && c === '/' && cn === '/' ) { // If we're currently inside a block, treat `//` as a start // of identifier. Else treat `//` as a start of a single-line // comment: parseSLComment ( css ) ; } // If current character is a double or single quote, it's a start // of a string: else if ( c === '"' || c === "'" ) { parseString ( css , c ) ; } // If current character is a space: else if ( c === ' ' ) { parseSpaces ( css ) ; } // If current character is `=`, it must be combined with next `=` else if ( c === '=' && cn === '=' ) { parseEquality ( css ) ; } // If we meet `!=`, this must be inequality else if ( c === '!' && cn === '=' ) { parseInequality ( css ) ; } // If current character is a punctuation mark: else if ( c in Punctuation ) { // Check for CRLF here or just LF if ( c === '\r' && cn === '\n' || c === '\n' ) { // If \r we know the next character is \n due to statement above // so we push a CRLF token type to the token list and importantly // skip the next character so as not to double count newlines or // columns etc if ( c === '\r' ) { pushToken ( TokenType . Newline , '\r\n' , col ) ; pos ++ ; // If CRLF skip the next character and push crlf token } else if ( c === '\n' ) { // If just a LF newline and not part of CRLF newline we can just // push punctuation as usual pushToken ( Punctuation [ c ] , c , col ) ; } ln ++ ; // Go to next line col = 0 ; // Reset the column count } else if ( c !== '\r' && c !== '\n' ) { // Handle all other punctuation and add to list of tokens pushToken ( Punctuation [ c ] , c , col ) ; } // Go to next line if ( c === ')' ) urlMode = false ; // Exit url mode else if ( c === '\t' && tabSize > 1 ) col += ( tabSize - 1 ) ; } // If current character is a decimal digit: else if ( isDecimalDigit ( c ) ) { parseDecimalNumber ( css ) ; } // If current character is anything else: else { parseIdentifier ( css ) ; } } return tokens ; }
Convert a CSS string to a list of tokens
@param {string} css CSS string
@returns {Array} List of tokens
@private
Convert a CSS string to a list of tokens