Examples of DfTableArray

Examples of ivory.core.data.stat.DfTableArray

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);

    DfTableArray dfs = new DfTableArray(new Path(env.getDfByIntData()), fs);

    Path termsFilePath = new Path(env.getIndexTermsData());
    Path termIDsFilePath = new Path(env.getIndexTermIdsData());
    Path idToTermFilePath = new Path(env.getIndexTermIdMappingData());

    DefaultFrequencySortedDictionary termIDMap = new DefaultFrequencySortedDictionary(termsFilePath, termIDsFilePath, idToTermFilePath, fs);

    for (int i=1; i<=200; i++) {
      System.out.println(String.format("%d\t%s\t%d", i, termIDMap.getTerm(i), dfs.getDf(i)));
    }
  }
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

135
136
137
138
139
140
141
142
143
144
145
      LOG.info(" - dl data: " + pathMapping.get(dlFile));

      try{
        dict = new DefaultFrequencySortedDictionary(pathMapping.get(termsFile),
            pathMapping.get(termidsFile), pathMapping.get(idToTermFile), FileSystem.getLocal(conf));
        dfTable = new DfTableArray(pathMapping.get(dfFile), FileSystem.getLocal(conf));
      } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException("Error loading Terms File for dictionary from "+localFiles[0]);
      }
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

209
210
211
212
213
214
215
216
217
218
219
    eScoreFn = (ScoringModel) new Bm25();
    eScoreFn.setAvgDocLength(lang2AvgSentLen.get(eLang));        //average sentence length = heuristic based on De-En data
    eScoreFn.setDocCount(env.readCollectionDocumentCount());

    dict = new DefaultFrequencySortedDictionary(new Path(env.getIndexTermsData()), new Path(env.getIndexTermIdsData()), new Path(env.getIndexTermIdMappingData()), fs);
    dfTable = new DfTableArray(new Path(env.getDfByTermData()), fs);

    //for backward compatibility
    //    String indexTermsFile = localFiles[12].toString();
    //    String dfTableFile = localFiles[0].toString();
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

483
484
485
486
487
488
489
    eScoreFn = (ScoringModel) new Bm25();
    eScoreFn.setAvgDocLength(lang2AvgSentLen.get(eLang));        //average sentence length = heuristic based on De-En data
    eScoreFn.setDocCount(env.readCollectionDocumentCount());

    dict = new DefaultFrequencySortedDictionary(new Path(env.getIndexTermsData()), new Path(env.getIndexTermIdsData()), new Path(env.getIndexTermIdMappingData()), localFs);
    dfTable = new DfTableArray(new Path(env.getDfByTermData()), localFs);
  }
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

155
156
157
158
159
160
161
162
163
164
165
        LOG.info(" - df data: " + pathMapping.get(dfFile));

        try{
          dict = new DefaultFrequencySortedDictionary(pathMapping.get(termsFile),
              pathMapping.get(termidsFile), pathMapping.get(idToTermFile), FileSystem.getLocal(conf));
          dfTable = new DfTableArray(pathMapping.get(dfFile), FileSystem.getLocal(conf));
        } catch (Exception e) {
          e.printStackTrace();
          throw new RuntimeException("Error loading Terms File for dictionary from "+localFiles[0]);
        }    
       
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

416
417
418
419
420
421
422
423
424
425
426
      } catch (IOException e) {
        e.printStackTrace();
      } 

      DefaultFrequencySortedDictionary dict = new DefaultFrequencySortedDictionary(new Path(env.getIndexTermsData()), new Path(env.getIndexTermIdsData()), new Path(env.getIndexTermIdMappingData()), fs2);
      DfTableArray dfTable = new DfTableArray(new Path(dfByIntFile), fs2);

      HMapIFW transDfTable = CLIRUtils.translateDFTable(eVocab_e2f, fVocab_e2f, en2DeProbs, dict, dfTable);

      SequenceFile.Writer writer = SequenceFile.createWriter(fs2, conf, new Path(transDfFile), IntWritable.class, FloatWritable.class);
      for(MapIF.Entry term : transDfTable.entrySet()){
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

111
112
113
114
115
116
117
118
119
120
121
      } catch (IOException e) {
        throw new RuntimeException("Local cache files not read properly.");
      }

      try {
        mDFTable = new DfTableArray(pathMapping.get(dfFile), FileSystem.getLocal(conf));
      } catch (IOException e1) {
        throw new RuntimeException("Error loading df table from " + localFiles[0]);
      }

      try {
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

129
130
131
132
133
134
135
136
137
138
139
      LOG.info(" - dl data: " + pathMapping.get(dlFile));

      try{
        dict = new DefaultFrequencySortedDictionary(pathMapping.get(termsFile),
            pathMapping.get(termidsFile), pathMapping.get(idToTermFile), FileSystem.getLocal(conf));
        dfTable = new DfTableArray(pathMapping.get(dfFile), FileSystem.getLocal(conf));
      } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException("Error loading Terms File for dictionary from "+localFiles[0]);
      }
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

160
161
162
163
164
165
166
167
168
169
170
        LOG.info(" - df data: " + pathMapping.get(dfFile));

        try{
          dict = new DefaultFrequencySortedDictionary(pathMapping.get(termsFile),
              pathMapping.get(termidsFile), pathMapping.get(idToTermFile), FileSystem.getLocal(conf));
          dfTable = new DfTableArray(pathMapping.get(dfFile), FileSystem.getLocal(conf));
        } catch (Exception e) {
          e.printStackTrace();
          throw new RuntimeException("Error loading Terms File for dictionary from "+localFiles[0]);
        }    
       
View Full Code Here

Examples of ivory.core.data.stat.DfTableArray

421
422
423
424
425
426
427
428
429
430
431
      } catch (IOException e) {
        e.printStackTrace();
      } 

      DefaultFrequencySortedDictionary dict = new DefaultFrequencySortedDictionary(new Path(env.getIndexTermsData()), new Path(env.getIndexTermIdsData()), new Path(env.getIndexTermIdMappingData()), fs2);
      DfTableArray dfTable = new DfTableArray(new Path(dfByIntFile), fs2);

      HMapIFW transDfTable = CLIRUtils.translateDFTable(eVocab_e2f, fVocab_e2f, en2DeProbs, dict, dfTable);

      SequenceFile.Writer writer = SequenceFile.createWriter(fs2, conf, new Path(transDfFile), IntWritable.class, FloatWritable.class);
      for(MapIF.Entry term : transDfTable.entrySet()){
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.