Package ivory.core.data.stat

Examples of ivory.core.data.stat.PrefixEncodedGlobalStats

53
54
55
56
57
58
59
60
61
62
63
        sLogger.info("4: " + localFiles[4]);
        sLogger.info("5: " + localFiles[5]);

        FileSystem fs = FileSystem.getLocal(job);

        gs = new PrefixEncodedGlobalStats(localFiles[0], fs);
        gs.loadDFStats(localFiles[1], fs);
        gs.loadCFStats(localFiles[2], fs);

        String indexPath = job.get("Ivory.IndexPath");
        sLogger.info("loading TermIdMap from " + indexPath);
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
336
337
        en2DeProbs = new TTable_monolithic_IFAs(fs2, new Path(e2fttableFile), true);
      } catch (IOException e) {
        e.printStackTrace();
      } 
      PrefixEncodedGlobalStats globalStatsMap;
      globalStatsMap = new PrefixEncodedGlobalStats(new Path(termsFile), fs2);
      globalStatsMap.loadDFStats(new Path(dfByTermFile), fs2);

      HMapIFW transDfTable = CLIRUtils.translateDFTable(eVocab_e2f, fVocab_e2f, en2DeProbs, globalStatsMap);

      SequenceFile.Writer writer = SequenceFile.createWriter(fs2, conf, new Path(transDfFile), IntWritable.class, FloatWritable.class);
      for(MapIF.Entry term : transDfTable.entrySet()){
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
      //for (int i = 0; i < 5; i++)
      //        sLogger.info ("localFiles [" + i + "]: " + localFiles [i]);

      try{
        globalStatsMap = new PrefixEncodedGlobalStats(localFiles[0], FileSystem.getLocal(conf));
      } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException("Error loading Terms File for global stats from "+localFiles[0]);
      }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
        sLogger.info("4: " + localFiles[4]);
        sLogger.info("5: " + localFiles[5]);

        FileSystem fs = FileSystem.getLocal(job);

        gs = new PrefixEncodedGlobalStats(localFiles[0], fs);
        gs.loadDFStats(localFiles[1], fs);
        gs.loadCFStats(localFiles[2], fs);

        String indexPath = job.get("Ivory.IndexPath");
        sLogger.info("loading TermIdMap from " + indexPath);
View Full Code Here
TOP

Related Classes of ivory.core.data.stat.PrefixEncodedGlobalStats

  • edu.umd.cloud9.io.pair.PairOfIntLong
  • ivory.core.index.DistributeGlobalStatsToPostings$MyMapper
  • ivory.core.preprocess.BuildTranslatedTermDocVectors$DataWriterMapper
  • ivory.core.preprocess.BuildWeightedTermDocVectors$MyMapper
  • ivory.core.RetrievalEnvironment
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.