Examples of ivory.core.data.stat.DocLengthTable2B

ivory.core.data.stat.DocLengthTable2B
Object that keeps track of the length of each document in the collection as a two-byte integer (shorts). Document lengths are measured in number of terms.

Document length data is stored in a serialized data file, in the following format:
- An integer that specifies the docno offset d, where d + 1 is the first docno in the collection.
- An integer that specifies the number of documents in the collection n.
- Exactly n shorts, one for each document in the collection.
Since the documents are numbered sequentially starting at d + 1, each short corresponds unambiguously to a particular document.
@author Jimmy Lin

          FileSystem fs = FileSystem.getLocal(job);
          RetrievalEnvironment re = new RetrievalEnvironment(job.get("Ivory.IndexPath"),
              fs);
          Path path = re.getDoclengthsData();
          sLogger.debug("Reading doclengths: " + path);
          mDocLengthTable = new DocLengthTable2B(path, fs);
        } else {
          Path[] localFiles = DistributedCache.getLocalCacheFiles(job);
          mDocLengthTable = new DocLengthTable2B(localFiles[0], FileSystem.getLocal(job));
        }
      } catch (Exception e) {
        throw new RuntimeException("Error initializing DocLengthTable!");
      }

View Full Code Here

    }


    // Read the table of doc lengths.
    if (loadDoclengths) {
      LOG.info("Loading doclengths table...");
      doclengths = new DocLengthTable2B(getDoclengthsData(), fs);
      LOG.info(" - Number of docs: " + doclengths.getDocCount());
      LOG.info(" - Avg. doc length: " + doclengths.getAvgDocLength());
      LOG.info("Done!");
    }
  }

View Full Code Here

    }


    // Read the table of doc lengths.
    if (loadDoclengths) {
      LOG.info("Loading doclengths table...");
      doclengths = new DocLengthTable2B(getDoclengthsData(), fs);
      LOG.info(" - Number of docs: " + doclengths.getDocCount());
      LOG.info(" - Avg. doc length: " + doclengths.getAvgDocLength());
      LOG.info("Done!");
    }
  }

View Full Code Here

        throw new RuntimeException("Error loading df table from "+localFiles[0]);
      }  
      
      try {
        if(shortDocLengths)
          mDLTable = new DocLengthTable2B(localFiles[2], FileSystem.getLocal(conf));
        else 
          mDLTable = new DocLengthTable4B(localFiles[2], FileSystem.getLocal(conf));
      } catch (IOException e1) {
        throw new RuntimeException("Error loading dl table from "+localFiles[2]);
      }

View Full Code Here

    }


    // Read the table of doc lengths.
    if (loadDoclengths) {
      LOG.info("Loading doclengths table...");
      doclengths = new DocLengthTable2B(getDoclengthsData(), fs);
      LOG.info(" - Number of docs: " + doclengths.getDocCount());
      LOG.info(" - Avg. doc length: " + doclengths.getAvgDocLength());
      LOG.info("Done!");
    }
  }

View Full Code Here


      sLogger.info("Global Stats table loaded successfully.");


      try {
        if(shortDocLengths)
          mDLTable = new DocLengthTable2B(localFiles[2], FileSystem.getLocal(conf));
        else 
          mDLTable = new DocLengthTable4B(localFiles[2], FileSystem.getLocal(conf));
      } catch (IOException e1) {
        throw new RuntimeException("Error loading dl table from "+localFiles[4]);
      }

View Full Code Here

        throw new RuntimeException("Error loading df table from " + localFiles[0]);
      }


      try {
        if (shortDocLengths)
          mDLTable = new DocLengthTable2B(pathMapping.get(dlFile), FileSystem.getLocal(conf));
        else
          mDLTable = new DocLengthTable4B(pathMapping.get(dlFile), FileSystem.getLocal(conf));
      } catch (IOException e1) {
        throw new RuntimeException("Error loading dl table from " + localFiles[2]);
      }

View Full Code Here

          FileSystem fs = FileSystem.getLocal(job);
          RetrievalEnvironment re = new RetrievalEnvironment(job.get("Ivory.IndexPath"),
              fs);
          Path path = re.getDoclengthsData();
          sLogger.debug("Reading doclengths: " + path);
          mDocLengthTable = new DocLengthTable2B(path, fs);
        } else {
          Path[] localFiles = DistributedCache.getLocalCacheFiles(job);
          mDocLengthTable = new DocLengthTable2B(localFiles[0], FileSystem.getLocal(job));
        }
      } catch (Exception e) {
        throw new RuntimeException("Error initializing DocLengthTable!");
      }

View Full Code Here


      LOG.info("Global Stats table loaded successfully.");


      try {
        if(shortDocLengths)
          mDLTable = new DocLengthTable2B(pathMapping.get(dlFile), FileSystem.getLocal(conf));
        else 
          mDLTable = new DocLengthTable4B(pathMapping.get(dlFile), FileSystem.getLocal(conf));
      } catch (IOException e1) {
        throw new RuntimeException("Error loading dl table from "+localFiles[4]);
      }

View Full Code Here

    }


    // Read the table of doc lengths.
    if (loadDoclengths) {
      LOG.info("Loading doclengths table...");
      doclengths = new DocLengthTable2B(getDoclengthsData(), fs);
      LOG.info(" - Number of docs: " + doclengths.getDocCount());
      LOG.info(" - Avg. doc length: " + doclengths.getAvgDocLength());
      LOG.info("Done!");
    }
  }

View Full Code Here

0 1

TOP

Related Classes of ivory.core.data.stat.DocLengthTable2B

ivory.core.preprocess.BuildWeightedIntDocVectors$MyMapper

ivory.core.preprocess.BuildWeightedTermDocVectors$MyMapper

ivory.core.RetrievalEnvironment

ivory.pwsim.PCP$MyMapper

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

java.io.InputStreamReader

java.io.BufferedReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.