wylie.js
function(str) {
	var tokens = [] // size = str.length + 2
	var i = 0;
	var maxlen = str.length;
	TOKEN:
	while (i < maxlen) {
		var c = str.charAt(i);
		var mlo = m_tokens_start.get(c);
		// if there are multi-char tokens starting with this char, try them
		if (mlo != null) {
			for (var len = mlo; len > 1; len--) {
				if (i <= maxlen - len) {
					var tr = str.substring(i, i + len);
					if (m_tokens.contains(tr)) {
						tokens.push(tr);
						i += len;
						continue TOKEN;
					}
				}
			}
		}
		// things starting with backslash are special
		if (c == '\\' && i <= maxlen - 2) {
			if (str.charAt(i + 1) == 'u' && i <= maxlen - 6) {
				tokens.push(str.substring(i, i + 6));		// \\uxxxx
				i += 6;
			} else if (str.charAt(i + 1) == 'U' && i <= maxlen - 10) {
				tokens.push(str.substring(i, i + 10));		// \\Uxxxxxxxx
				i += 10;
			} else {
				tokens.push(str.substring(i, i + 2));		// \\x
				i += 2;
			}
			continue TOKEN;
		}
		// otherwise just take one char
		tokens.push(c.toString());
		i += 1;
	}
	return tokens;
}
function(str) {
	var tokens = [] // size = str.length + 2
	var i = 0;
	var maxlen = str.length;
	TOKEN:
	while (i < maxlen) {
		var c = str.charAt(i);
		var mlo = m_tokens_start.get(c);
		// if there are multi-char tokens starting with this char, try them
		if (mlo != null) {
			for (var len = mlo; len > 1; len--) {
				if (i <= maxlen - len) {
					var tr = str.substring(i, i + len);
					if (m_tokens.contains(tr)) {
						tokens.push(tr);
						i += len;
						continue TOKEN;
					}
				}
			}
		}
		// things starting with backslash are special
		if (c == '\\' && i <= maxlen - 2) {
			if (str.charAt(i + 1) == 'u' && i <= maxlen - 6) {
				tokens.push(str.substring(i, i + 6));		// \\uxxxx
				i += 6;
			} else if (str.charAt(i + 1) == 'U' && i <= maxlen - 10) {
				tokens.push(str.substring(i, i + 10));		// \\Uxxxxxxxx
				i += 10;
			} else {
				tokens.push(str.substring(i, i + 2));		// \\x
				i += 2;
			}
			continue TOKEN;
		}
		// otherwise just take one char
		tokens.push(c.toString());
		i += 1;
	}
	return tokens;
}
function ( str ) { var tokens = [ ] // size = str.length + 2 var i = 0 ; var maxlen = str . length ; TOKEN : while ( i < maxlen ) { var c = str . charAt ( i ) ; var mlo = m_tokens_start . get ( c ) ; // if there are multi-char tokens starting with this char, try them if ( mlo != null ) { for ( var len = mlo ; len > 1 ; len -- ) { if ( i <= maxlen - len ) { var tr = str . substring ( i , i + len ) ; if ( m_tokens . contains ( tr ) ) { tokens . push ( tr ) ; i += len ; continue TOKEN ; } } } } // things starting with backslash are special if ( c == '\\' && i <= maxlen - 2 ) { if ( str . charAt ( i + 1 ) == 'u' && i <= maxlen - 6 ) { tokens . push ( str . substring ( i , i + 6 ) ) ; // \\uxxxx i += 6 ; } else if ( str . charAt ( i + 1 ) == 'U' && i <= maxlen - 10 ) { tokens . push ( str . substring ( i , i + 10 ) ) ; // \\Uxxxxxxxx i += 10 ; } else { tokens . push ( str . substring ( i , i + 2 ) ) ; // \\x i += 2 ; } continue TOKEN ; } // otherwise just take one char tokens . push ( c . toString ( ) ) ; i += 1 ; } return tokens ; }
split a string into Wylie tokens; make sure there is room for at least one null element at the end of the array
split a string into Wylie tokens ; make sure there is room for at least one null element at the end of the array