lib/calais-entity-extractor.js
function(url, cb) {
        var calais = this;

        if (!calais.validateOptions())
            return cb({}, 'Bad options');

        //Make sure we were given a URL.
        if (this._undefinedOrNull(url) || typeof url != 'string' || url.length == 0)
            return cb({}, 'No URL given.');

        //Make sure it's a valid URL.
        if (!(/^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url)))
            return cb({}, 'Bad URL');


        request(url, function(error, response, html) {
            if (error)
                return cb({}, error);

            //Limts to just under 100kb....
            html = html.substring(0, 95000);
            
            //We can upload the html directly to Calais if we set the contentType as text/html
            var params = {
                'Host'                   : calais.options.apiHost,
                'x-ag-access-token'      : calais.apiKey,
                'x-calais-language'      : calais.options.language,
                'Content-Type'           : 'text/html',
                'Accept'                 : 'application/json',
                'Content-Length'         : html.length,
                'OutputFormat'           : 'application/json'
            };

            var options = {
                uri    : 'https://' + calais.options.apiHost + calais.options.apiPath,
                method : 'POST',
                body   : html,
                headers: params
            };


            request(options, function(error, response, calaisData) {

                if (error)
                    return cb({}, error);

                if (response === undefined) {
                    return cb({}, 'Undefined Calais response');
                } else if (response.statusCode === 200) {
                    if (!calaisData)
                        return cb({}, calais.errors);

                    try {
                        // parse to a Javascript object if requested
                        var result = JSON.parse(calaisData);
                        result = (typeof result === 'string') ? JSON.parse(result) : result;

                        var parsedResult;
                        if (calais.options.parseData) {
                            parsedResult = calais._parseCalaisData(result, calais.options.minConfidence);
                        } else {
                            parsedResult = result;
                        }

                        return cb(parsedResult, calais.errors);
                    } catch(e) {
                        return cb({}, e);
                    }

                } else
                    return cb({}, 'Request error: ' + (typeof response === 'string' ? response : JSON.stringify(response)));


            });

        });

    }
function(url, cb) {
        var calais = this;

        if (!calais.validateOptions())
            return cb({}, 'Bad options');

        //Make sure we were given a URL.
        if (this._undefinedOrNull(url) || typeof url != 'string' || url.length == 0)
            return cb({}, 'No URL given.');

        //Make sure it's a valid URL.
        if (!(/^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url)))
            return cb({}, 'Bad URL');


        request(url, function(error, response, html) {
            if (error)
                return cb({}, error);

            //Limts to just under 100kb....
            html = html.substring(0, 95000);
            
            //We can upload the html directly to Calais if we set the contentType as text/html
            var params = {
                'Host'                   : calais.options.apiHost,
                'x-ag-access-token'      : calais.apiKey,
                'x-calais-language'      : calais.options.language,
                'Content-Type'           : 'text/html',
                'Accept'                 : 'application/json',
                'Content-Length'         : html.length,
                'OutputFormat'           : 'application/json'
            };

            var options = {
                uri    : 'https://' + calais.options.apiHost + calais.options.apiPath,
                method : 'POST',
                body   : html,
                headers: params
            };


            request(options, function(error, response, calaisData) {

                if (error)
                    return cb({}, error);

                if (response === undefined) {
                    return cb({}, 'Undefined Calais response');
                } else if (response.statusCode === 200) {
                    if (!calaisData)
                        return cb({}, calais.errors);

                    try {
                        // parse to a Javascript object if requested
                        var result = JSON.parse(calaisData);
                        result = (typeof result === 'string') ? JSON.parse(result) : result;

                        var parsedResult;
                        if (calais.options.parseData) {
                            parsedResult = calais._parseCalaisData(result, calais.options.minConfidence);
                        } else {
                            parsedResult = result;
                        }

                        return cb(parsedResult, calais.errors);
                    } catch(e) {
                        return cb({}, e);
                    }

                } else
                    return cb({}, 'Request error: ' + (typeof response === 'string' ? response : JSON.stringify(response)));


            });

        });

    }
function ( url , cb ) { var calais = this ; if ( ! calais . validateOptions ( ) ) return cb ( { } , 'Bad options' ) ; //Make sure we were given a URL. if ( this . _undefinedOrNull ( url ) || typeof url != 'string' || url . length == 0 ) return cb ( { } , 'No URL given.' ) ; //Make sure it's a valid URL. if ( ! ( / ^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$ / i . test ( url ) ) ) return cb ( { } , 'Bad URL' ) ; request ( url , function ( error , response , html ) { if ( error ) return cb ( { } , error ) ; //Limts to just under 100kb.... html = html . substring ( 0 , 95000 ) ; //We can upload the html directly to Calais if we set the contentType as text/html var params = { 'Host' : calais . options . apiHost , 'x-ag-access-token' : calais . apiKey , 'x-calais-language' : calais . options . language , 'Content-Type' : 'text/html' , 'Accept' : 'application/json' , 'Content-Length' : html . length , 'OutputFormat' : 'application/json' } ; var options = { uri : 'https://' + calais . options . apiHost + calais . options . apiPath , method : 'POST' , body : html , headers : params } ; request ( options , function ( error , response , calaisData ) { if ( error ) return cb ( { } , error ) ; if ( response === undefined ) { return cb ( { } , 'Undefined Calais response' ) ; } else if ( response . statusCode === 200 ) { if ( ! calaisData ) return cb ( { } , calais . errors ) ; try { // parse to a Javascript object if requested var result = JSON . parse ( calaisData ) ; result = ( typeof result === 'string' ) ? JSON . parse ( result ) : result ; var parsedResult ; if ( calais . options . parseData ) { parsedResult = calais . _parseCalaisData ( result , calais . options . minConfidence ) ; } else { parsedResult = result ; } return cb ( parsedResult , calais . errors ) ; } catch ( e ) { return cb ( { } , e ) ; } } else return cb ( { } , 'Request error: ' + ( typeof response === 'string' ? response : JSON . stringify ( response ) ) ) ; } ) ; } ) ; }
Extract tags and entities from a given URL. We download the HTML from the URL, and submit
that to Calais using the extractFromText function

@param url The URL to analyze.
@param cb The callback function, of form function(result, error)
Extract tags and entities from a given URL . We download the HTML from the URL and submit that to Calais using the extractFromText function