Examples of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.store.CompoundFileDirectory
Class for accessing a compound stream. This class implements a directory, but is limited to only read operations. Directory methods that would normally modify data throw an exception.
All files belonging to a segment have the same name with varying extensions. The extensions correspond to the different file formats used by the {@link Codec}. When using the Compound File format these files are collapsed into a single .cfs file (except for the {@link LiveDocsFormat}, with a corresponding .cfe file indexing its sub-files.
Files:
- .cfs: An optional "virtual" file consisting of all the other index files for systems that frequently run out of file handles.
- .cfe: The "virtual" compound file's entry table holding all entries in the corresponding .cfs file.
Description:
- Compound (.cfs) --> Header, FileData ^FileCount
- Compound Entry Table (.cfe) --> Header, FileCount, <FileName, DataOffset, DataLength> ^FileCount
- Header --> {@link CodecUtil#writeHeader CodecHeader}
- FileCount --> {@link DataOutput#writeVInt VInt}
- DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}
- FileName --> {@link DataOutput#writeString String}
- FileData --> raw file data
Notes:
- FileCount indicates how many files are contained in this compound file. The entry table that follows has that many entries.
- Each directory entry contains a long pointer to the start of this file's data section, the files length, and a String with that file's name.
@lucene.experimental


  // note: intentionally ignores seg suffix
  Lucene40DocValuesWriter(SegmentWriteState state, String filename, String legacyKey) throws IOException {
    this.state = state;
    this.legacyKey = legacyKey;
    this.dir = new CompoundFileDirectory(state.directory, filename, state.context, true);
  }

View Full Code Here

    for (String file : dir.listAll()) {
      if (file.equals(IndexFileNames.SEGMENTS_GEN)) {
        continue; // segments.gen has no header, thats ok
      }
      if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
        CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
        checkHeaders(cfsDir); // recurse into cfs
        cfsDir.close();
      }
      IndexInput in = null;
      boolean success = false;
      try {
        in = dir.openInput(file, newIOContext(random()));

View Full Code Here

    
    boolean success = false;


    try {
      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
        d = storeCFSReader = new CompoundFileDirectory(si.dir, 
            IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
      } else {
        storeCFSReader = null;
      }
      String idxName = IndexFileNames.segmentFileName(segment, "", VECTORS_INDEX_EXTENSION);

View Full Code Here

        // 2.x segment, and an IndexFormatTooOldException will be thrown,
        // which is what we want.
        Directory dir = directory;
        if (Lucene3xSegmentInfoFormat.getDocStoreOffset(si) != -1) {
          if (Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
            dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
                Lucene3xSegmentInfoFormat.getDocStoreSegment(si), "",
                Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), IOContext.READONCE, false);
          }
        } else if (si.getUseCompoundFile()) {
          dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
              si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE, false);
        }


        try {
          Lucene3xStoredFieldsReader.checkCodeVersion(dir, Lucene3xSegmentInfoFormat.getDocStoreSegment(si));

View Full Code Here

      infoStream.message("IW", "create compound file " + fileName);
    }
    assert Lucene3xSegmentInfoFormat.getDocStoreOffset(info) == -1;
    // Now merge all added files
    Collection<String> files = info.files();
    CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true);
    IOException prior = null;
    try {
      for (String file : files) {
        directory.copy(cfsDir, file, file, context);
        checkAbort.work(directory.fileLength(file));

View Full Code Here

  private final AtomicLong ramBytesUsed;


  Lucene40DocValuesReader(SegmentReadState state, String filename, String legacyKey) throws IOException {
    this.state = state;
    this.legacyKey = legacyKey;
    this.dir = new CompoundFileDirectory(state.directory, filename, state.context, false);
    ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOf(getClass()));
  }

View Full Code Here

    final int size = si.getDocCount();
    boolean success = false;
    fieldInfos = fn;
    try {
      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
        d = storeCFSReader = new CompoundFileDirectory(si.dir, 
            IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
      } else {
        storeCFSReader = null;
      }
      fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);

View Full Code Here

  static FieldInfos readFieldInfos(SegmentCommitInfo info) throws IOException {
    final Directory dir;
    final boolean closeDir;
    if (info.getFieldInfosGen() == -1 && info.info.getUseCompoundFile()) {
      // no fieldInfos gen and segment uses a compound file
      dir = new CompoundFileDirectory(info.info.dir,
          IndexFileNames.segmentFileName(info.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION),
          IOContext.READONCE,
          false);
      closeDir = true;
    } else {

View Full Code Here


    boolean success = false;
    
    try {
      if (si.info.getUseCompoundFile()) {
        cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
      } else {
        cfsReader = null;
        cfsDir = dir;
      }

View Full Code Here

    public void testSingleFile() throws IOException {
        int data[] = new int[] { 0, 1, 10, 100 };
        for (int i=0; i<data.length; i++) {
            String name = "t" + data[i];
            createSequenceFile(dir, name, (byte) 0, data[i]);
            CompoundFileDirectory csw = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), true);
            dir.copy(csw, name, name, newIOContext(random()));
            csw.close();


            CompoundFileDirectory csr = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), false);
            IndexInput expected = dir.openInput(name, newIOContext(random()));
            IndexInput actual = csr.openInput(name, newIOContext(random()));
            assertSameStreams(name, expected, actual);
            assertSameSeekBehavior(name, expected, actual);
            expected.close();
            actual.close();
            csr.close();
        }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader

org.apache.lucene.codecs.lucene3x.Lucene3xStoredFieldsReader

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsFormat

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesWriter

org.apache.lucene.index.CompoundFileExtractor

org.apache.lucene.index.IndexWriter

org.apache.lucene.index.SegmentCoreReaders

org.apache.lucene.index.SegmentReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.