[Summary view] [Print] [Text view]

   1  <?php
   2  /*

   3   * Limb PHP Framework

   4   *

   5   * @link http://limb-project.com

   6   * @copyright  Copyright &copy; 2004-2007 BIT(http://bit-creative.com)

   7   * @license    LGPL http://www.gnu.org/copyleft/lesser.html

   8   */
   9  
  10  /**

  11   * @package search

  12   * @version $Id: indexer.php 5989 2007-06-13 13:08:11Z pachanga $

  13   */
  14  if(!isset($argv[1]))
  15    die("index starting uri not specified!\n");
  16  
  17  $path = $_SERVER['LIMB_PROJECT_DIR'];
  18  require_once($path . '/setup.php');
  19  
  20  lmb_require('limb/net/src/lmbUri.class.php');
  21  lmb_require('limb/web_spider/src/lmbWebSpider.class.php');
  22  lmb_require('limb/web_spider/src/lmbUriFilter.class.php');
  23  lmb_require('limb/web_spider/src/lmbContentTypeFilter.class.php');
  24  lmb_require('limb/web_spider/src/lmbSearchIndexingObserver.class.php');
  25  lmb_require('limb/search/src/indexer/lmbFullTextSearchIndexer.class.php');
  26  lmb_require('limb/search/src/indexer/lmbSearchTextNormalizer.class.php');
  27  lmb_require('limb/web_spider/src/lmbUriNormalizer.class.php');
  28  
  29  $uri = new lmbUri($argv[1]);
  30  
  31  $indexer = new lmbFullTextSearchIndexer(new lmbSearchTextNormalizer());
  32  $indexer->useNOINDEX();
  33  
  34  $observer = new lmbSearchIndexingObserver($indexer);
  35  
  36  $content_type_filter = new lmbContentTypeFilter();
  37  $content_type_filter->allowContentType('text/html');
  38  
  39  $uri_filter = new lmbUriFilter();
  40  $uri_filter->allowHost($uri->getHost());
  41  $uri_filter->allowProtocol('http');
  42  $uri_filter->allowPathRegex('~.*~');
  43  
  44  $normalizer = new lmbUriNormalizer();
  45  $normalizer->stripQueryItem('PHPSESSID');
  46  
  47  $spider = new lmbWebSpider();
  48  $spider->setContentTypeFilter($content_type_filter);
  49  $spider->setUriFilter($uri_filter);
  50  $spider->setUriNormalizer($normalizer);
  51  $spider->registerObserver($observer);
  52  
  53  $spider->crawl($uri);
  54  
  55  ?>