Examples of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.store.CompoundFileDirectory
Class for accessing a compound stream. This class implements a directory, but is limited to only read operations. Directory methods that would normally modify data throw an exception.
All files belonging to a segment have the same name with varying extensions. The extensions correspond to the different file formats used by the {@link Codec}. When using the Compound File format these files are collapsed into a single .cfs file (except for the {@link LiveDocsFormat}, with a corresponding .cfe file indexing its sub-files.
Files:
- .cfs: An optional "virtual" file consisting of all the other index files for systems that frequently run out of file handles.
- .cfe: The "virtual" compound file's entry table holding all entries in the corresponding .cfs file.
Description:
- Compound (.cfs) --> Header, FileData ^FileCount
- Compound Entry Table (.cfe) --> Header, FileCount, <FileName, DataOffset, DataLength> ^FileCount
- Header --> {@link CodecUtil#writeHeader CodecHeader}
- FileCount --> {@link DataOutput#writeVInt VInt}
- DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}
- FileName --> {@link DataOutput#writeString String}
- FileData --> raw file data
Notes:
- FileCount indicates how many files are contained in this compound file. The entry table that follows has that many entries.
- Each directory entry contains a long pointer to the start of this file's data section, the files length, and a String with that file's name.
@lucene.experimental

   }
   
   
  public void testAppend() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    int size = 5 + random().nextInt(128);
    for (int j = 0; j < 2; j++) {
      IndexOutput os = csw.createOutput("seg_" + j + "_foo.txt", newIOContext(random()));
      for (int i = 0; i < size; i++) {
        os.writeInt(i*j);
      }
      os.close();
      String[] listAll = newDir.listAll();
      assertEquals(1, listAll.length);
      assertEquals("d.cfs", listAll[0]);
    }
    createSequenceFile(dir, "d1", (byte) 0, 15);
    dir.copy(csw, "d1", "d1", newIOContext(random()));
    String[] listAll = newDir.listAll();
    assertEquals(1, listAll.length);
    assertEquals("d.cfs", listAll[0]);
    csw.close();
    CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    for (int j = 0; j < 2; j++) {
      IndexInput openInput = csr.openInput("seg_" + j + "_foo.txt", newIOContext(random()));
      assertEquals(size * 4, openInput.length());
      for (int i = 0; i < size; i++) {
        assertEquals(i*j, openInput.readInt());
      }


      openInput.close();


    }
    IndexInput expected = dir.openInput("d1", newIOContext(random()));
    IndexInput actual = csr.openInput("d1", newIOContext(random()));
    assertSameStreams("d1", expected, actual);
    assertSameSeekBehavior("d1", expected, actual);
    expected.close();
    actual.close();
    csr.close();
    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testAppendTwice() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    createSequenceFile(newDir, "d1", (byte) 0, 15);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    out.close();
    assertEquals(1, csw.listAll().length);
    assertEquals("d.xyz", csw.listAll()[0]);
   
    csw.close();


    CompoundFileDirectory cfr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(1, cfr.listAll().length);
    assertEquals("d.xyz", cfr.listAll()[0]);
    cfr.close();
    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testEmptyCFS() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    csw.close();


    CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    assertEquals(0, csr.listAll().length);
    csr.close();


    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testReadNestedCFP() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    CompoundFileDirectory nested = new CompoundFileDirectory(newDir, "b.cfs", newIOContext(random()), true);
    IndexOutput out = nested.createOutput("b.xyz", newIOContext(random()));
    IndexOutput out1 = nested.createOutput("b_1.xyz", newIOContext(random()));
    out.writeInt(0);
    out1.writeInt(1);
    out.close();
    out1.close();
    nested.close();
    newDir.copy(csw, "b.cfs", "b.cfs", newIOContext(random()));
    newDir.copy(csw, "b.cfe", "b.cfe", newIOContext(random()));
    newDir.deleteFile("b.cfs");
    newDir.deleteFile("b.cfe");
    csw.close();
    
    assertEquals(2, newDir.listAll().length);
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    
    assertEquals(2, csw.listAll().length);
    nested = new CompoundFileDirectory(csw, "b.cfs", newIOContext(random()), false);
    
    assertEquals(2, nested.listAll().length);
    IndexInput openInput = nested.openInput("b.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    openInput = nested.openInput("b_1.xyz", newIOContext(random()));
    assertEquals(1, openInput.readInt());
    openInput.close();
    nested.close();
    csw.close();
    newDir.close();
  }

View Full Code Here

    newDir.close();
  }
  
  public void testDoubleClose() throws IOException {
    Directory newDir = newDirectory();
    CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
    IndexOutput out = csw.createOutput("d.xyz", newIOContext(random()));
    out.writeInt(0);
    out.close();
    
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), false);
    IndexInput openInput = csw.openInput("d.xyz", newIOContext(random()));
    assertEquals(0, openInput.readInt());
    openInput.close();
    csw.close();
    // close a second time - must have no effect according to Closeable
    csw.close();
    
    newDir.close();
    
  }

View Full Code Here

      IndexOutput out = d.createOutput("file." + fileIdx, newIOContext(random()));
      out.writeByte((byte) fileIdx);
      out.close();
    }
    
    final CompoundFileDirectory cfd = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), true);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      final String fileName = "file." + fileIdx;
      d.copy(cfd, fileName, fileName, newIOContext(random()));
    }
    cfd.close();


    final IndexInput[] ins = new IndexInput[FILE_COUNT];
    final CompoundFileDirectory cfr = new CompoundFileDirectory(d, "c.cfs", newIOContext(random()), false);
    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx] = cfr.openInput("file." + fileIdx, newIOContext(random()));
    }


    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      assertEquals((byte) fileIdx, ins[fileIdx].readByte());
    }


    for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
      ins[fileIdx].close();
    }
    cfr.close();
    d.close();
  }

View Full Code Here

  
  // checks that we can open all files returned by listAll!
  private void checkFiles(Directory dir) throws IOException {
    for (String file : dir.listAll()) {
      if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
        CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
        checkFiles(cfsDir); // recurse into cfs
        cfsDir.close();
      }
      IndexInput in = null;
      boolean success = false;
      try {
        in = dir.openInput(file, newIOContext(random()));

View Full Code Here

    public void testSingleFile() throws IOException {
        int data[] = new int[] { 0, 1, 10, 100 };
        for (int i=0; i<data.length; i++) {
            String name = "t" + data[i];
            createSequenceFile(dir, name, (byte) 0, data[i]);
            CompoundFileDirectory csw = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), true);
            dir.copy(csw, name, name, newIOContext(random()));
            csw.close();


            CompoundFileDirectory csr = new CompoundFileDirectory(dir, name + ".cfs", newIOContext(random()), false);
            IndexInput expected = dir.openInput(name, newIOContext(random()));
            IndexInput actual = csr.openInput(name, newIOContext(random()));
            assertSameStreams(name, expected, actual);
            assertSameSeekBehavior(name, expected, actual);
            expected.close();
            actual.close();
            csr.close();
        }
    }

View Full Code Here

     */
    public void testTwoFiles() throws IOException {
        createSequenceFile(dir, "d1", (byte) 0, 15);
        createSequenceFile(dir, "d2", (byte) 0, 114);


        CompoundFileDirectory csw = new CompoundFileDirectory(dir, "d.cfs", newIOContext(random()), true);
        dir.copy(csw, "d1", "d1", newIOContext(random()));
        dir.copy(csw, "d2", "d2", newIOContext(random()));
        csw.close();


        CompoundFileDirectory csr = new CompoundFileDirectory(dir, "d.cfs", newIOContext(random()), false);
        IndexInput expected = dir.openInput("d1", newIOContext(random()));
        IndexInput actual = csr.openInput("d1", newIOContext(random()));
        assertSameStreams("d1", expected, actual);
        assertSameSeekBehavior("d1", expected, actual);
        expected.close();
        actual.close();


        expected = dir.openInput("d2", newIOContext(random()));
        actual = csr.openInput("d2", newIOContext(random()));
        assertSameStreams("d2", expected, actual);
        assertSameSeekBehavior("d2", expected, actual);
        expected.close();
        actual.close();
        csr.close();
    }

View Full Code Here

        createRandomFile(dir, "onetwothree", 100);
        createRandomFile(dir, segment + ".notIn", 50);
        createRandomFile(dir, segment + ".notIn2", 51);


        // Now test
        CompoundFileDirectory csw = new CompoundFileDirectory(dir, "test.cfs", newIOContext(random()), true);
        final String data[] = new String[] {
            ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
            ".big4", ".big5", ".big6", ".big7"
        };
        for (int i=0; i<data.length; i++) {
            String fileName = segment + data[i];
            dir.copy(csw, fileName, fileName, newIOContext(random()));
        }
        csw.close();


        CompoundFileDirectory csr = new CompoundFileDirectory(dir, "test.cfs", newIOContext(random()), false);
        for (int i=0; i<data.length; i++) {
            IndexInput check = dir.openInput(segment + data[i], newIOContext(random()));
            IndexInput test = csr.openInput(segment + data[i], newIOContext(random()));
            assertSameStreams(data[i], check, test);
            assertSameSeekBehavior(data[i], check, test);
            test.close();
            check.close();
        }
        csr.close();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.store.CompoundFileDirectory

org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader

org.apache.lucene.codecs.lucene3x.Lucene3xStoredFieldsReader

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsFormat

org.apache.lucene.codecs.lucene3x.Lucene3xTermVectorsReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesReader

org.apache.lucene.codecs.lucene40.Lucene40DocValuesWriter

org.apache.lucene.index.CompoundFileExtractor

org.apache.lucene.index.IndexWriter

org.apache.lucene.index.SegmentCoreReaders

org.apache.lucene.index.SegmentReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.