Examples of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.store.CompoundFileDirectory
Class for accessing a compound stream. This class implements a directory, but is limited to only read operations. Directory methods that would normally modify data throw an exception.
All files belonging to a segment have the same name with varying extensions. The extensions correspond to the different file formats used by the {@link Codec}. When using the Compound File format these files are collapsed into a single .cfs file (except for the {@link LiveDocsFormat}, with a corresponding .cfe file indexing its sub-files.
Files:
- .cfs: An optional "virtual" file consisting of all the other index files for systems that frequently run out of file handles.
- .cfe: The "virtual" compound file's entry table holding all entries in the corresponding .cfs file.
Description:
- Compound (.cfs) --> Header, FileData ^FileCount
- Compound Entry Table (.cfe) --> Header, FileCount, <FileName, DataOffset, DataLength> ^FileCount
- Header --> {@link CodecUtil#writeHeader CodecHeader}
- FileCount --> {@link DataOutput#writeVInt VInt}
- DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}
- FileName --> {@link DataOutput#writeString String}
- FileData --> raw file data
Notes:
- FileCount indicates how many files are contained in this compound file. The entry table that follows has that many entries.
- Each directory entry contains a long pointer to the start of this file's data section, the files length, and a String with that file's name.
@lucene.experimental

    newDir.close();
  }
  
  public void testAppendTwice() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    createSequenceFile(newDir, "d1", (byte) 0, 15);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    try {
      newDir.copy(csw, "d1", "d1", newIOContext(random()));
      fail("file does already exist");
    } catch (IllegalArgumentException e) {
      //
    }
    out.close();
    assertEquals(1, csw.listAll().length);
    assertEquals("d.xyz", csw.listAll()[0]);
   
    csw.close();


    CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(1, cfr.listAll().length);
    assertEquals("d.xyz", cfr.listAll()[0]);
    cfr.close();
    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testEmptyCFS() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    csw.close();


    CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(0, csr.listAll().length);
    csr.close();


    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testReadNestedCFP() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    CompoundFileDirectory nested = new CompoundFileDirectory(newDir, "b.cfs", newIOContext(random()), true);
    IndexOutput out = nested.createOutput("b.xyz", newIOContext(random()));
    IndexOutput out1 = nested.createOutput("b_1.xyz", newIOContext(random()));
    out.writeInt(0);
    out1.writeInt(1);
    out.close();
    out1.close();
    nested.close();
    newDir.copy(csw, "b.cfs", "b.cfs", newIOContext(random()));
    newDir.copy(csw, "b.cfe", "b.cfe", newIOContext(random()));
    newDir.deleteFile("b.cfs");
    newDir.deleteFile("b.cfe");
    csw.close();
    
    assertEquals(2, newDir.listAll().length);
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    
    assertEquals(2, csw.listAll().length);
    nested = new CompoundFileDirectory(csw, "b.cfs", newIOContext(random()), false);
    
    assertEquals(2, nested.listAll().length);
    IndexInput openInput = nested.openInput("b.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    openInput = nested.openInput("b_1.xyz", newIOContext(random()));
    assertEquals(1, openInput.readInt());
    openInput.close();
    nested.close();
    csw.close();
    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testDoubleClose() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    out.close();
    
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    IndexInput openInput = csw.openInput("d.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    newDir.close();
    
  }

View Full Code Here

      IndexOutput out = d.createOutput("file." + fileIdx, newIOContext(random()));
      out.writeByte((byte) fileIdx);
      out.close();
    }
    
    final CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), true);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      final String fileName = "file." + fileIdx;
      d.copy(cfd, fileName, fileName, newIOContext(random()));
    }
    cfd.close();


    final IndexInput[] ins = new IndexInput[FILE_COUNT];
    final CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), false);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx] = cfr.openInput("file." + fileIdx, newIOContext(random()));
    }


    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      assertEquals((byte) fileIdx, ins[fileIdx].readByte());
    }


    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx].close();
    }
    cfr.close();
    d.close();
  }

View Full Code Here

  
  // checks that we can open all files returned by listAll!
  private void checkFiles(Directory dir) throws IOException {
    for (String file : dir.listAll()) {
      if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
        CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
        checkFiles(cfsDir); // recurse into cfs
        cfsDir.close();
      }
      IndexInput in = null;
      boolean success = false;
      try {
        in = dir.openInput(file, newIOContext(random()));

View Full Code Here

  static FieldInfos readFieldInfos(SegmentCommitInfo info) throws IOException {
    final Directory dir;
    final boolean closeDir;
    if (info.getFieldInfosGen() == -1 && info.info.getUseCompoundFile()) {
      // no fieldInfos gen and segment uses a compound file
      dir = new CompoundFileDirectory(info.info.dir,
          IndexFileNames.segmentFileName(info.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION),
          IOContext.READONCE,
          false);
      closeDir = true;
    } else {

View Full Code Here


    boolean success = false;
    
    try {
      if (si.info.getUseCompoundFile()) {
        cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
      } else {
        cfsReader = null;
        cfsDir = dir;
      }

View Full Code Here

    
    boolean success = false;


    try {
      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
        d = storeCFSReader = new CompoundFileDirectory(si.dir, 
            IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
      } else {
        storeCFSReader = null;
      }
      String idxName = IndexFileNames.segmentFileName(segment, "", VECTORS_INDEX_EXTENSION);

View Full Code Here

    final int size = si.getDocCount();
    boolean success = false;
    fieldInfos = fn;
    try {
      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
        d = storeCFSReader = new CompoundFileDirectory(si.dir, 
            IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
      } else {
        storeCFSReader = null;
      }
      fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader

org.apache.lucene.codecs.lucene3x.Lucene3xStoredFieldsReader

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsFormat

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesWriter

org.apache.lucene.index.CompoundFileExtractor

org.apache.lucene.index.IndexWriter

org.apache.lucene.index.SegmentCoreReaders

org.apache.lucene.index.SegmentReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.