Package com.flaptor.util

Examples of com.flaptor.util.Config

66
67
68
69
70
71
72
73
   */
  public BatchIndexManager(IndexDescriptor indexDescriptor, File indexDirectory) {
        index = Index.createIndex(indexDirectory);
        index.setIndexDescriptor(indexDescriptor);
        writer= index.getWriter();
        Config config = Config.getConfig("indexer.properties");
    docIdName = config.getString("docIdName");
  }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
   
    /**
     * Class initializer.
     */
    public FetchdataProcessor() throws IOException {
        Config config = Config.getConfig("crawler.properties");
        maxDistance = config.getInt("max.distance");
        IS_HOTSPOT = config.getString("hotspot.tag");
        hotspots = new UrlPatterns(config.getString("hotspot.file"));
        recordParents = config.getBoolean("record.parents");
        modules = ModulesManager.getInstance();
        int cpus = Runtime.getRuntime().availableProcessors();
        workerCount = (int) Math.ceil(cpus * config.getFloat("workers.per.cpu"));
        logger.info("FetchDocument processor using " + workerCount + " workers on " + cpus + " cpus");
    }
View Full Code Here
283
284
285
286
287
288
289
290
291
292
   */
  private static void doCommonTasks(File componentDir, ComponentConfig componentConfig) throws IOException {
        File confDir = new File(componentDir, "conf");
        File logsDir = new File(componentDir, "logs");
      FileUtil.createIfDoesntExist(logsDir, true, true);
    Config commonProperties = Config.getConfig("common.properties");
    commonProperties.set("port.base", String.valueOf(componentConfig.installOnBasePort));
    commonProperties.set("baseDir", componentConfig.installOnBaseDir.getPath());
    commonProperties.modifyOnDisk(new File(confDir,"common.properties"));
  }
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
305
  private static void installSearcher(String distDir, File installBasePath, SearcherConfig searcherConfig, CacheServerConfig cacheServerConfig) throws IOException {
        logger.info("installing searcher");
        TarUtil.untarFile(FileUtil.getExistingFile(distDir+"/searcher.tgz", true, false, false), installBasePath);
    doCommonTasks(new File(installBasePath,"/searcher"), searcherConfig);

    Config properties = Config.getConfig("searcher.properties");
   
    properties.set("searcher.cachedVersion.showLink", String.valueOf(cacheServerConfig.install));
    properties.set("searcher.cachedVersion.cacheServerHost", cacheServerConfig.externalHostName + ":" + cacheServerConfig.installOnBasePort);
    properties.modifyOnDisk(new File(installBasePath,"/searcher/conf/searcher.properties"));
   
  }
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
  private static void installCrawler(String distDir, File installBasePath, CrawlerConfig crawlerConfig, IndexerConfig indexerConfig) throws IOException {
        logger.info("installing crawler");
        TarUtil.untarFile(FileUtil.getExistingFile(distDir+"/crawler.tgz", true, false, false), installBasePath);
    doCommonTasks(new File(installBasePath,"/crawler"), crawlerConfig);

    Config indexerModuleProperties = Config.getConfig("indexerModule.properties");
    indexerModuleProperties.set("remoteRmiIndexer.host", indexerConfig.installOnHost + ":" + indexerConfig.installOnBasePort);
    indexerModuleProperties.modifyOnDisk(new File(installBasePath,"/crawler/conf/indexerModule.properties"));
   
    configureCrawler(new File(installBasePath, "crawler"), crawlerConfig);
  }
View Full Code Here
341
342
343
344
345
346
347
348
349
350
  private static void installIndexer(String distDir, File installBasePath, IndexerConfig indexerConfig, SearcherConfig searcherConfig) throws IOException {
    logger.info("installing indexer");
    TarUtil.untarFile(FileUtil.getExistingFile(distDir+"/indexer.tgz", true, false, false), installBasePath);
    doCommonTasks(new File(installBasePath,"/indexer"), indexerConfig);

    Config indexerProperties = Config.getConfig("indexer.properties");
    indexerProperties.set("IndexLibrary.rsyncAccessString", indexerConfig.installOnHost);
    indexerProperties.set("IndexLibrary.remoteIndexUpdaters", searcherConfig.installOnHost + ":" + searcherConfig.installOnBasePort);
    indexerProperties.modifyOnDisk(new File(installBasePath,"/indexer/conf/indexer.properties"));
  }
View Full Code Here
352
353
354
355
356
357
358
359
360
361
  private static void installCacheServer(String distDir, File installBasePath, CacheServerConfig cacheServerConfig, CrawlerConfig crawlerConfig) throws IOException {
        logger.info("installing cache-server");
        TarUtil.untarFile(FileUtil.getExistingFile(distDir+"/cache-server.tgz", true, false, false), installBasePath);
    doCommonTasks(new File(installBasePath,"/cache-server"), cacheServerConfig);
       
        Config indexerProperties = Config.getConfig("multiCache.properties");
    indexerProperties.set("multiCache.hosts", crawlerConfig.installOnHost + ":" + crawlerConfig.installOnBasePort);
    indexerProperties.modifyOnDisk(new File(installBasePath,"/cache-server/conf/multiCache.properties"));

  }
View Full Code Here
369
370
371
372
373
374
375
376
377
    if (searcherConfig.install) hosts += searcherConfig.installOnHost + ":" + (searcherConfig.installOnBasePort + PortUtil.getOffset("clustering.rpc.searcher")) + ":" + searcherConfig.installOnBaseDir+"/searcher" + ":" + searcherConfig.clusteringType;
    if (indexerConfig.install) hosts += (hosts.length() == 0?"":",") + indexerConfig.installOnHost + ":" + (indexerConfig.installOnBasePort + PortUtil.getOffset("clustering.rpc.indexer")) + ":" + searcherConfig.installOnBaseDir+"/indexer" + ":" + indexerConfig.clusteringType;
    if (crawlerConfig.install) hosts += (hosts.length() == 0?"":",") + crawlerConfig.installOnHost + ":" + (crawlerConfig.installOnBasePort + PortUtil.getOffset("clustering.rpc.crawler")) + ":" + searcherConfig.installOnBaseDir+"/crawler" + ":" + crawlerConfig.clusteringType;
    if (cacheServerConfig.install) hosts += (hosts.length() == 0?"":",") + cacheServerConfig.installOnHost + ":" + (cacheServerConfig.installOnBasePort + PortUtil.getOffset("clustering.rpc.cacheServer")) + ":" + searcherConfig.installOnBaseDir+"/cache-server" + ":" + cacheServerConfig.clusteringType;
   
    Config clusteringProperties = Config.getConfig("clustering.properties");
    clusteringProperties.set("clustering.nodes", hosts);
    clusteringProperties.modifyOnDisk(new File(installBasePath,"/clustering-web/conf/clustering.properties"));
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
        super(name, globalConfig);
       
        attributesToLog = loadTags("attributes.to.log");
        tagsToLog = loadTags("tags.to.log");
        categoriesToLog = loadTags("categories.to.log");
        Config mdlConfig = getModuleConfig();
        logText= mdlConfig.getBoolean("log.text");
        logEmmited= mdlConfig.getBoolean("log.emmited");

        String outputFileName= mdlConfig.getString("log.file.name");

        if (null == outputFileName || 0 == outputFileName.length() ||
                outputFileName.equalsIgnoreCase("stdout")){
            out= System.out;
        } else if (outputFileName.equalsIgnoreCase("stderr")){
View Full Code Here
413
414
415
416
417
418
419
420
421
422
423
424
425
      BlockingQueue<IndexSearcher> searcherPool;
        private final long timeout;

      public IndexRepository(final Index index) throws SearcherException {
            this.index = index;
          Config conf = Config.getConfig("searcher.properties");
          if (conf.getBoolean("compositeSearcher.useTrafficLimiting")) {
            timeout = conf.getInt("searcher.trafficLimiting.maxTimeInQueue");
            if (timeout < 0L) {
                throw new IllegalArgumentException("timeout is a negative number (" + timeout + ")");
            }
            logger.info("IndexRepository constructor: setting the max time to wait for an available index searcher to " + timeout);
        } else {
View Full Code Here
TOP

Related Classes of com.flaptor.util.Config

  • com.flaptor.hounder.cache.HttpCacheServer$CacheHandler
  • com.flaptor.hounder.classifier.bayes.BayesCalculatorTest
  • com.flaptor.hounder.classifier.WhyBean
  • com.flaptor.hounder.cluster.MultiSearcher
  • com.flaptor.hounder.crawler.bean.CrawlerBean
  • com.flaptor.hounder.crawler.Crawler
  • com.flaptor.hounder.crawler.CrawlerProgress
  • com.flaptor.hounder.crawler.CrawlerTest
  • com.flaptor.hounder.crawler.FetchdataProcessor
  • com.flaptor.hounder.crawler.FetcherTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.