Node/core/lib/dialogs/PromptRecognizers.js
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
function tokenize ( text ) { var tokens = [ ] ; if ( text && text . length > 0 ) { var token = '' ; for ( var i = 0 ; i < text . length ; i ++ ) { var chr = text [ i ] ; if ( breakingChars . indexOf ( chr ) >= 0 ) { if ( token . length > 0 ) { tokens . push ( token ) ; } token = '' ; } else { token += chr ; } } if ( token . length > 0 ) { tokens . push ( token ) ; } } return tokens ; }
Breaks a string of text into an array of tokens.
Breaks a string of text into an array of tokens .