Package com.flaptor.util

Examples of com.flaptor.util.Config

44
45
46
47
48
49
50
51
52
53
54
55
56
    private Index currentIndex;
    private Map<String,Index> indexes;
    private Set<String> knownClusters;

    public IndexComposer() {
        Config config = Config.getConfig("composer.properties");
        baseDir = new File(config.getString("IndexComposer.baseDir"));
        String[] clusters = config.getStringArray("IndexComposer.clusters");
        indexes = new HashMap<String,Index>(clusters.length);
        knownClusters = new HashSet<String>(clusters.length);

        for (String cluster: clusters) {
           knownClusters.add(cluster) ;
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    private final Booster booster;

    public BoostModule (String moduleName, Config globalConfig) {
        super(moduleName, globalConfig);
        ABoostCondition.Type conditionType = null ;
        Config mdlConfig = getModuleConfig();
        String conditionString = mdlConfig.getString("boost.condition");
        if ("url".equals(conditionString)) conditionType = ABoostCondition.Type.UrlPattern;
        if ("keyword".equals(conditionString)) conditionType = ABoostCondition.Type.Keyword;
        if ("urlandkeyword".equals(conditionString)) conditionType = ABoostCondition.Type.UrlPatternAndKeyword;
        ABoostCondition condition = ABoostCondition.getBoostCondition(mdlConfig,conditionType);

        ABoostMethod.Type methodType = null;
        String methodString = mdlConfig.getString("boost.method");
        if ("doc".equals(methodString)) methodType = ABoostMethod.Type.Doc;
        if ("field".equals(methodString)) methodType = ABoostMethod.Type.Field;
        if ("keyword".equals(methodString)) methodType = ABoostMethod.Type.Keyword;
        ABoostMethod method = ABoostMethod.getBoostMethod(mdlConfig,methodType);

        ABoostValue.Type valueType = null;
        valueType = ABoostValue.Type.None;
        String valueString = mdlConfig.getString("boost.value");
        if ("constant".equals(valueString)) valueType = ABoostValue.Type.Constant;
        if ("attribute".equals(valueString)) valueType = ABoostValue.Type.Attribute;
        ABoostValue value = ABoostValue.getBoostValue(mdlConfig,valueType);

        booster = new Booster(condition,value,method);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
     *       A LocalIndexUpdater that will receive the retrieved indexes.
     */
    public IndexUpdatesListener(LocalIndexUpdater lib) {
        this.library = lib;

        Config config = Config.getConfig("searcher.properties");
        this.indexDir = Config.getConfig("common.properties").getString("baseDir")
                  + File.separator + config.getString("searcher.dir")
                  + File.separator + "indexes";
        int fetchIndexPort = PortUtil.getPort("post.new.index");
        server = new RmiServer(fetchIndexPort);
        server.addHandler(RmiServer.DEFAULT_SERVICE_NAME, this);
        server.start();
View Full Code Here
119
120
121
122
123
124
125
126
                        "<newFoo>" +
                            "<text><xsl:value-of select=\"bar\"/></text>" +
                        "</newFoo>" +
                    "</xsl:template>" +
                "</xsl:stylesheet>");
        Config conf = Config.getConfig("indexer.properties");
        conf.set("XsltModulFile", file.getAbsolutePath());
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
     * Class initializer. The fetchserver works with a provided fetcher.
     * @param fetcher the provided fetcher, null if it should be read from the config.
     * @param crawler the crawler that will provide fetchlists and consume fetchdata.
     */
    public FetchServer (IFetcher fetcher, Crawler crawler) throws Exception {
        Config config = Config.getConfig("crawler.properties");
        if (null == fetcher) {
            String className = config.getString("fetcher.plugin");
            fetcher = (IFetcher)Class.forName(className).getConstructor(new Class[]{}).newInstance(new Object[]{});
        }
        retryOnDisconnect = config.getBoolean("retry.fetch.on.disconnect");
        this.fetcher = fetcher;
        this.crawler = crawler;
        serverThread = new ServerThread();
        serverThread.setDaemon(true);
    }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
   * @todo review the way exceptions are handled.
   */
    private ModulesManager() {
        modules = new ArrayList<IProcessorModule>();

        Config config = Config.getConfig("crawler.properties");
        String moduleNames = config.getString("modules");
        if (null == moduleNames || "".equals(moduleNames)) {
            logger.warn("No crawler modules defined");
            return;
        }
        String[] singleModules = moduleNames.split("\\|");
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
     * Constructor.
     */
    public MultiIndexer() {

        // get the indexer stubs.
        Config config = Config.getConfig("multiIndexer.properties");
        String[] hosts = config.getStringArray("indexer.hosts");
        for (int i = 0; i < hosts.length; i++) {
          Pair<String, Integer> host = PortUtil.parseHost(hosts[i]);
            indexers.add(new RmiIndexerStub(host.last(), host.first()));
        }

        hashFunction = new Hash(indexers.size());

        useXslt = config.getBoolean("multiIndexer.useXslt");
        if (useXslt) {
            try {
                xsltModule = new XsltModule();
                logger.info("MultiIndexer will be using XsltModule");
            } catch (Exception e) {
                logger.error("Constructor: while instantiating XsltModule: " + e,e);
                throw new RuntimeException(e);
            }
        } else {
            xsltModule = null;
            logger.info("MultiIndexer will NOT be using XsltModule");
        }

        if (config.getBoolean("clustering.enable")) {
          int port = PortUtil.getPort("clustering.rpc.indexer");
        nodeListener = new NodeListener(port, config);
            ControllerModule.addModuleListener(nodeListener, new ControllableImplementation());
        indexerMonitoredNode = IndexerMonitoredNode.getInstance();
        MonitorModule.addModuleListener(nodeListener, IndexerMonitoredNode.getInstance());
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
     * Class initializer.
     * Uses the default fetcher and indexer.
     */
    public PageDBTrimmer () throws IOException {
        init(false);
        Config config = Config.getConfig("crawler.properties");
        maxDistance = config.getInt("max.distance");
        hotspots = new UrlPatterns(config.getString("hotspot.file"));
        discoveryFrontSize = config.getLong("discovery.front.size");
        stocasticDiscoveryFrontLine = config.getBoolean("discovery.front.stocastic");
        cyclesBetweenDiscoveryWaves = config.getInt("cycles.between.discovery.waves");
        pagedbSizeLimit = config.getInt("pagedb.size.limit");
    }
View Full Code Here
442
443
444
445
446
447
448
449
450
451
452
453
        Execute.close(hotspots);
    }

    public void test(String name) throws Exception {
        Crawler crawler = new Crawler();
        Config config = Config.getConfig("crawler.properties");
        UrlPatterns hotspots = new UrlPatterns(config.getString("hotspot.file"));
        PageFilter pageFilter = new PageFilter(0, new int[]{3}, 0L, 0L, 0L);
        PageDB destPageDB = new PageDB(name+".test");
        destPageDB.open(PageDB.WRITE+PageDB.UNSORTED);
        PageDB pdb = new PageDB(name);
        pdb.open(PageDB.READ);
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
   
    public OutLinksCountryModule(String name, Config globalConfig) {
        super(name, globalConfig);
        String sitesFile;
       
        Config mdlConfig = getModuleConfig();
        regexp[OUTLINKS_1]= mdlConfig.getString("outlinks.sites.regexp.1").trim();
        sitesFile= mdlConfig.getString("outlinks.sites.file.1");
        knownSites[OUTLINKS_1]= new HashSet<String>();
        if (null == sitesFile || "".equals(sitesFile)) {
            logger.warn("There is no outlinks.sites.file.1 file defined.");
        } else {
            logger.info("Loading " + sitesFile + " as sites.file.1");
            FileUtil.fileToSet(null, sitesFile, knownSites[OUTLINKS_1],logger);
        }

        regexp[OUTLINKS_2]= mdlConfig.getString("outlinks.sites.regexp.2").trim();
        sitesFile= mdlConfig.getString("outlinks.sites.file.2");
        knownSites[OUTLINKS_2]= new HashSet<String>();
        if (null == sitesFile || "".equals(sitesFile)) {
            logger.warn("There is no outlinks.sites.file.2 file defined.");
        } else {
            logger.info("Loading " + sitesFile + " as sites.file.2");
            FileUtil.fileToSet(null, sitesFile, knownSites[OUTLINKS_2],logger);
        }

        regexp[OUTLINKS_IGNORE]= mdlConfig.getString("outlinks.sites.regexp.ignore").trim();
        sitesFile= mdlConfig.getString("outlinks.sites.file.ignore");
        knownSites[OUTLINKS_IGNORE]= new HashSet<String>();
        if (null == sitesFile || "".equals(sitesFile)) {
            logger.warn("There is no outlinks.sites.file.ignore file defined.");
        } else {
            logger.info("Loading " + sitesFile + " as sites.file.ignore");
View Full Code Here
TOP

Related Classes of com.flaptor.util.Config

  • com.flaptor.hounder.cache.HttpCacheServer$CacheHandler
  • com.flaptor.hounder.classifier.bayes.BayesCalculatorTest
  • com.flaptor.hounder.classifier.WhyBean
  • com.flaptor.hounder.cluster.MultiSearcher
  • com.flaptor.hounder.crawler.bean.CrawlerBean
  • com.flaptor.hounder.crawler.Crawler
  • com.flaptor.hounder.crawler.CrawlerProgress
  • com.flaptor.hounder.crawler.CrawlerTest
  • com.flaptor.hounder.crawler.FetchdataProcessor
  • com.flaptor.hounder.crawler.FetcherTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.