src/sap.ui.core/src/sap/ui/model/odata/v4/lib/_Parser.js
function tokenize(sOption) {
		var iAt = 1, // The token's position for error messages; we count starting with 1
			sId,
			aMatches,
			sNext = sOption,
			iOffset,
			oToken,
			aTokens = [],
			sValue;

		while (sNext.length) {
			aMatches = rToken.exec(sNext);
			iOffset = 0;
			if (aMatches) {
				sValue = aMatches[0];
				if (aMatches[7]) {
					sId = "OPTION";
				} else if (aMatches[6] || aMatches[4]) {
					sId = "VALUE";
				} else if (aMatches[5]) {
					sId = "PATH";
					if (sValue === "false" || sValue === "true" || sValue === "null") {
						sId = "VALUE";
					} else if (sValue === "not") {
						sId = "not";
						aMatches = rNot.exec(sNext);
						if (aMatches) {
							sValue = aMatches[0];
						}
					}
				} else if (aMatches[3]) { // a %-escaped delimiter
					sId = unescape(aMatches[3]);
				} else if (aMatches[2]) { // an operator
					sId = aMatches[2];
					iOffset = aMatches[1].length;
				} else { // a delimiter
					sId = aMatches[0];
				}
				if (sId === '"') {
					sId = "VALUE";
					sValue = tokenizeDoubleQuotedString(sNext, sOption, iAt);
				} else if (sId === "'") {
					sId = "VALUE";
					sValue = tokenizeSingleQuotedString(sNext, sOption, iAt);
				}
				oToken = {
					at : iAt + iOffset,
					id : sId,
					value : sValue
				};
			} else {
				throw new SyntaxError("Unknown character '" + sNext[0] + "' at " + iAt + ": "
					+ sOption);
			}
			sNext = sNext.slice(sValue.length);
			iAt += sValue.length;
			aTokens.push(oToken);
		}

		return aTokens;
	}
function tokenize(sOption) {
		var iAt = 1, // The token's position for error messages; we count starting with 1
			sId,
			aMatches,
			sNext = sOption,
			iOffset,
			oToken,
			aTokens = [],
			sValue;

		while (sNext.length) {
			aMatches = rToken.exec(sNext);
			iOffset = 0;
			if (aMatches) {
				sValue = aMatches[0];
				if (aMatches[7]) {
					sId = "OPTION";
				} else if (aMatches[6] || aMatches[4]) {
					sId = "VALUE";
				} else if (aMatches[5]) {
					sId = "PATH";
					if (sValue === "false" || sValue === "true" || sValue === "null") {
						sId = "VALUE";
					} else if (sValue === "not") {
						sId = "not";
						aMatches = rNot.exec(sNext);
						if (aMatches) {
							sValue = aMatches[0];
						}
					}
				} else if (aMatches[3]) { // a %-escaped delimiter
					sId = unescape(aMatches[3]);
				} else if (aMatches[2]) { // an operator
					sId = aMatches[2];
					iOffset = aMatches[1].length;
				} else { // a delimiter
					sId = aMatches[0];
				}
				if (sId === '"') {
					sId = "VALUE";
					sValue = tokenizeDoubleQuotedString(sNext, sOption, iAt);
				} else if (sId === "'") {
					sId = "VALUE";
					sValue = tokenizeSingleQuotedString(sNext, sOption, iAt);
				}
				oToken = {
					at : iAt + iOffset,
					id : sId,
					value : sValue
				};
			} else {
				throw new SyntaxError("Unknown character '" + sNext[0] + "' at " + iAt + ": "
					+ sOption);
			}
			sNext = sNext.slice(sValue.length);
			iAt += sValue.length;
			aTokens.push(oToken);
		}

		return aTokens;
	}
function tokenize ( sOption ) { var iAt = 1 , // The token's position for error messages; we count starting with 1 sId , aMatches , sNext = sOption , iOffset , oToken , aTokens = [ ] , sValue ; while ( sNext . length ) { aMatches = rToken . exec ( sNext ) ; iOffset = 0 ; if ( aMatches ) { sValue = aMatches [ 0 ] ; if ( aMatches [ 7 ] ) { sId = "OPTION" ; } else if ( aMatches [ 6 ] || aMatches [ 4 ] ) { sId = "VALUE" ; } else if ( aMatches [ 5 ] ) { sId = "PATH" ; if ( sValue === "false" || sValue === "true" || sValue === "null" ) { sId = "VALUE" ; } else if ( sValue === "not" ) { sId = "not" ; aMatches = rNot . exec ( sNext ) ; if ( aMatches ) { sValue = aMatches [ 0 ] ; } } } else if ( aMatches [ 3 ] ) { // a %-escaped delimiter sId = unescape ( aMatches [ 3 ] ) ; } else if ( aMatches [ 2 ] ) { // an operator sId = aMatches [ 2 ] ; iOffset = aMatches [ 1 ] . length ; } else { // a delimiter sId = aMatches [ 0 ] ; } if ( sId === '"' ) { sId = "VALUE" ; sValue = tokenizeDoubleQuotedString ( sNext , sOption , iAt ) ; } else if ( sId === "'" ) { sId = "VALUE" ; sValue = tokenizeSingleQuotedString ( sNext , sOption , iAt ) ; } oToken = { at : iAt + iOffset , id : sId , value : sValue } ; } else { throw new SyntaxError ( "Unknown character '" + sNext [ 0 ] + "' at " + iAt + ": " + sOption ) ; } sNext = sNext . slice ( sValue . length ) ; iAt += sValue . length ; aTokens . push ( oToken ) ; } return aTokens ; }
Splits the option string into an array of tokens.

@param {string} sOption The option string
@returns {object[]} The array of tokens.
Splits the option string into an array of tokens .