lib/search.js
function getLinkContents(err, queuedLink) {
  // no link found
  if (!queuedLink) {
    return;
  }
  // retrieve https directly
  var method = (queuedLink.uri.toString().indexOf('https') === 0) ? utils.retrieveHTTPS : utils.retrieve;

  method(queuedLink.uri, function(err, data) {
    contentLib.indexContentItem({ uri: ''+queuedLink.uri, content: data}, {isHTML: true, member: queuedLink.queued.member}, function(err, res, cItem) {
      if (err || !cItem) {
        GLOBAL.error('getLinkContents failed', err, res);
        return;
      }
      // if it recently gained content queue links
      if (cItem.state !== utils.states.content.queued && cItem.previousState === utils.states.content.queued) {
        GLOBAL.info('queueing links', cItem.uri, 'relevance', cItem.queued.relevance);
        queueLinks(cItem);
      }
      GLOBAL.debug('scraped', err, data ? data.length : 'no content');
    });
  });
}
function getLinkContents(err, queuedLink) {
  // no link found
  if (!queuedLink) {
    return;
  }
  // retrieve https directly
  var method = (queuedLink.uri.toString().indexOf('https') === 0) ? utils.retrieveHTTPS : utils.retrieve;

  method(queuedLink.uri, function(err, data) {
    contentLib.indexContentItem({ uri: ''+queuedLink.uri, content: data}, {isHTML: true, member: queuedLink.queued.member}, function(err, res, cItem) {
      if (err || !cItem) {
        GLOBAL.error('getLinkContents failed', err, res);
        return;
      }
      // if it recently gained content queue links
      if (cItem.state !== utils.states.content.queued && cItem.previousState === utils.states.content.queued) {
        GLOBAL.info('queueing links', cItem.uri, 'relevance', cItem.queued.relevance);
        queueLinks(cItem);
      }
      GLOBAL.debug('scraped', err, data ? data.length : 'no content');
    });
  });
}
function getLinkContents ( err , queuedLink ) { // no link found if ( ! queuedLink ) { return ; } // retrieve https directly var method = ( queuedLink . uri . toString ( ) . indexOf ( 'https' ) === 0 ) ? utils . retrieveHTTPS : utils . retrieve ; method ( queuedLink . uri , function ( err , data ) { contentLib . indexContentItem ( { uri : '' + queuedLink . uri , content : data } , { isHTML : true , member : queuedLink . queued . member } , function ( err , res , cItem ) { if ( err || ! cItem ) { GLOBAL . error ( 'getLinkContents failed' , err , res ) ; return ; } // if it recently gained content queue links if ( cItem . state !== utils . states . content . queued && cItem . previousState === utils . states . content . queued ) { GLOBAL . info ( 'queueing links' , cItem . uri , 'relevance' , cItem . queued . relevance ) ; queueLinks ( cItem ) ; } GLOBAL . debug ( 'scraped' , err , data ? data . length : 'no content' ) ; } ) ; } ) ; }
retrieve and process any queued link
retrieve and process any queued link