Package com.flaptor.util

Examples of com.flaptor.util.Config

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
        for (int i = 0; i < numServers; i++) {
        //    setUpIndexer(i);
            sb.append("cluster"+i + ",");
        }
        Config.getConfig("common.properties").set("port.base","10000");
        Config composerConfig = Config.getConfig("composer.properties");
        tmpDir = com.flaptor.util.FileUtil.createTempDir("junit", ".tmp").getAbsolutePath();
        Config.getConfig("common.properties").set("baseDir", tmpDir);
        composerConfig.set("IndexComposer.clusters",sb.substring(0,sb.length() - 1));
        composerConfig.set("IndexComposer.baseDir",tmpDir);

        Config.getConfig("indexer.properties").set("Writer.fields", "");
        Config.getConfig("indexer.properties").set("clustering.enable","false");

        indexers = new ArrayList<Indexer>();
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    }


    private void setUpIndexer(int numServer) throws Exception {
        String tmpDir = com.flaptor.util.FileUtil.createTempDir("junit", ".tmp").getAbsolutePath();
        Config indexerConfig = Config.getConfig("indexer.properties");
        // Configuration
        // indexer:
        Config.getConfig("common.properties").set("baseDir", tmpDir);
        indexerConfig.set("IndexManager.updateInterval", "1000");
        indexerConfig.set("IndexManager.indexDescriptor", "0of1@cluster"+numServer);
        indexerConfig.set("IndexLibrary.remoteIndexUpdaters","127.0.0.1:10000");
        indexerConfig.set("Indexer.modules", "com.flaptor.hounder.indexer.Writer");
        indexerConfig.set("Indexer.fields", "content");
        indexerConfig.set("docIdName", "docId");
        indexerConfig.set("Indexer.maxQueueSize", "100");
        indexerConfig.set("IndexLibrary.rsyncAccessString","");

        Indexer indexer = new Indexer();
        addTemplate.setAttribute("content", "content " + numServer);
        indexer.index(addTemplate.toString());       
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111

    @TestInfo(testType = TestInfo.TestType.INTEGRATION)
    public void testMerge() throws Exception{
       
        Config searcherConfig = Config.getConfig("searcher.properties");
        searcherConfig.set("searcher.dir", "searcher");
        IndexComposer composer = new IndexComposer();
        new IndexUpdatesListener(composer); // dont keep reference
       
        for (int i = 0; i < numServers; i++) {
            setUpIndexer(i);
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    tmpDir = com.flaptor.util.FileUtil.createTempDir("junit", ".tmp").getAbsolutePath();
    // Configuration
        Config.getConfig("common.properties").set("baseDir", tmpDir);
        Config.getConfig("common.properties").set("port.base", "10000");
    // indexer:
    Config indexerConfig = Config.getConfig("indexer.properties");
    indexerConfig.set("IndexManager.updateInterval", "4000");
        indexerConfig.set("IndexLibrary.remoteIndexUpdaters","localhost:10000");
        indexerConfig.set("IndexLibrary.rsyncAccessString","");
    indexerConfig.set("Indexer.modules", "com.flaptor.hounder.indexer.Writer");
    indexerConfig.set("Indexer.fields", "content");
    indexerConfig.set("docIdName", "docId");
    indexerConfig.set("Indexer.maxQueueSize", "100");
    indexerConfig.set("Writer.fields","");
    indexerConfig.set("clustering.enable","false");

    // searcher:
    Config searcherConfig = Config.getConfig("searcher.properties");
        searcherConfig.set("compositeSearcher.useSpellCheckSuggestQuery", "false");
    searcherConfig.set("QueryParser.searchFields", "content");
    searcherConfig.set("QueryParser.searchFieldWeights", "1.0f");
    searcherConfig.set("QueryParser.nonTokenizedFields", "");
    searcherConfig.set("ReloadableIndexSearcher.minTimeBetweenIndexes", "1000");
    searcherConfig.set("ReloadableIndexSearcher.sleepTime", "1000");
    searcherConfig.set("clustering.enable", "false");
        searcherConfig.set("SimilarityForwarder.scorers","");

    //TODO see why it crashes if snippets are set
    searcherConfig.set("Searcher.generateSnippets", "false");

        indexer = new Indexer();
        searcher = new CompositeSearcher();
        baseSearcher = (Searcher)searcher.getBaseSearcher();
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        assertNotNull(suggestor.suggest(new LazyParsedQuery("content")));
        assertNotNull(suggestor.suggest(new LazyParsedQuery("contenta")));
        assertNotNull(suggestor.suggest(new LazyParsedQuery("contetb")));

        Config searcherConfig = Config.getConfig("searcher.properties");
        searcherConfig.set("compositeSearcher.useSpellCheckSuggestQuery", "true");
        searcherConfig.set("searcher.suggestQuerySearcher.dictionaryDir", spellDir.getFile().getAbsolutePath());
        searcher = new CompositeSearcher();

        GroupedSearchResults res = searcher.search(new LazyParsedQuery("contentb"), 0, 10, null, 20, null, null);
        assertEquals(1, res.totalGroupsEstimation());
        res = searcher.search(new LazyParsedQuery("content"), 0, 10, null, 20, null, null);
        assertEquals(0, res.totalGroupsEstimation());
        assertNotNull(res.getSuggestedQuery());
        res = searcher.search(res.getSuggestedQuery(), 0, 10, null, 20, null, null);
        assertEquals(1, res.totalGroupsEstimation());

        searcherConfig.set("compositeSearcher.useSuggestQuery", "false");
    }
View Full Code Here
433
434
435
436
437
438
439
440
441
442
443
444
    private void trySearcherStartup (ArrayList<Pair<String,String>> indexes, boolean[] indexUse, boolean cleanup, File wd) throws Exception {
        File indexesDir = new File(tmpDir,"/searcher/indexes");
        FileUtil.deleteDir(indexesDir);
        indexesDir.mkdir();
        Config searcherConfig = Config.getConfig("searcher.properties");
        searcherConfig.set("IndexLibrary.cleanupOnStartup",cleanup?"true":"false");
        for (int i = 0; i<3; i++) {
            if (indexUse[i]) {
                Execute.sleep(1010); // needed to ensure the order of the last modification time of the index dirs (has to be greater than 1 sec).
                CommandUtil.execute("cp -lr "+indexes.get(i).last()+" "+indexes.get(i).first(), wd);
                (new File(indexes.get(i).first())).setLastModified(System.currentTimeMillis());
View Full Code Here
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
    }

    @TestInfo(testType = TestInfo.TestType.SYSTEM,
            requiresPort = {10000, 10001, 10010, 10011, 10012})
    public void testPayloads() throws Exception {
        Config searcherConfig = Config.getConfig("searcher.properties");

        indexer.index(addSmallPayload);
        indexer.index(addBigPayload);

        Execute.sleep(10000);


        // perform query
        GroupedSearchResults gsr = searcher.search(new TermQuery("content", "contenta"), 0, 10, new NoGroup(), 1, null, null);
        assertEquals(gsr.groups(),2);

        // check that small payload is first, if we do not use payloads.
        assertEquals(gsr.getGroup(0).last().get(0).get("docId"),"small");
        assertEquals(gsr.getGroup(1).last().get(0).get("docId"),"big");


        // override config, so payloads are used
        searcherConfig.set("SimilarityForwarder.scorers","payload:com.flaptor.hounder.searcher.payload.DatePayloadScorer");

        // restart searcher
        stopSearcher();
        searcher = new CompositeSearcher();
        Execute.sleep(10000);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
                      "pass.through.on.missing.tags=";
        TestUtils.setConfig(tmpConfig,
                "page.text.max.length=250\n" +
                "hotspot.tag=TAG_IS_HOTSPOT\n" +
                "emitdoc.tag=TAG_IS_INDEXABLE");
        Config cfg = TestUtils.getConfig();

        //TestUtils.writeFile(MOCK_LANG + POS_PROP, mockDo);
        TestUtils.writeFile(MOD_NAME + POS_PROP, mockDo);
        ldm= new LanguageDetectionModule(MOD_NAME, cfg);
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
    /**
     * @see TestCase#setUp()
     */
    public void setUp() {
        Config config = Config.getConfig("indexer.properties");
        config.set("FieldFormatChecker.fields","date:long");
        mod = new FieldFormatCheckerModule();
        filterOutput("Dropping");
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
   
    public void setUp() throws Exception {
        String sitesAr= SA_1 + "\n" + SA_2 + "\n";
        TestUtils.writeFile(sitesArFile, sitesAr);

        Config modcfg = Config.getConfig(MOD_NAME+POS_PROP);
        modcfg.set("pass.through.on.tags","");
        modcfg.set("pass.through.on.missing.tags","");
        modcfg.set("threshold.value","0.25");
        modcfg.set("on.above.threshold.set.tags","");
        modcfg.set("on.above.threshold.unset.tags","");
        modcfg.set("on.below.threshold.set.tags","");
        modcfg.set("on.below.threshold.unset.tags","");
        modcfg.set("outlinks.sites.regexp.1","^.*\\.ar$");
        modcfg.set("outlinks.sites.file.1",sitesArFile);
        modcfg.set("outlinks.sites.regexp.2","^.*\\.\\p{Alpha}{2}?$");
        modcfg.set("outlinks.sites.file.2","");
        modcfg.set("outlinks.sites.regexp.ignore","^.*\\.(tv|ws|cc)$");
        modcfg.set("outlinks.sites.file.ignore","");
       
        TestUtils.setConfig(tmpConfig,
                "page.text.max.length=250\n" +
                "hotspot.tag=TAG_IS_HOTSPOT\n" +
                "emitdoc.tag=TAG_IS_INDEXABLE");
        Config cfg = TestUtils.getConfig();
        module = new OutLinksCountryModule(MOD_NAME, cfg);
    }
View Full Code Here
TOP

Related Classes of com.flaptor.util.Config

  • com.flaptor.hounder.cache.HttpCacheServer$CacheHandler
  • com.flaptor.hounder.classifier.bayes.BayesCalculatorTest
  • com.flaptor.hounder.classifier.WhyBean
  • com.flaptor.hounder.cluster.MultiSearcher
  • com.flaptor.hounder.crawler.bean.CrawlerBean
  • com.flaptor.hounder.crawler.Crawler
  • com.flaptor.hounder.crawler.CrawlerProgress
  • com.flaptor.hounder.crawler.CrawlerTest
  • com.flaptor.hounder.crawler.FetchdataProcessor
  • com.flaptor.hounder.crawler.FetcherTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.