Examples of BaseDirectoryWrapper


Examples of org.apache.lucene.store.BaseDirectoryWrapper

      if (VERBOSE) {
        System.out.println("TEST: index " + unsupportedNames[i]);
      }
      File oldIndxeDir = _TestUtil.getTempDir(unsupportedNames[i]);
      _TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
      BaseDirectoryWrapper dir = newFSDirectory(oldIndxeDir);
      // don't checkindex, these are intentionally not supported
      dir.setCheckIndexOnClose(false);

      IndexReader reader = null;
      IndexWriter writer = null;
      try {
        reader = DirectoryReader.open(dir);
        fail("DirectoryReader.open should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
      } finally {
        if (reader != null) reader.close();
        reader = null;
      }

      try {
        writer = new IndexWriter(dir, newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random())));
        fail("IndexWriter creation should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
        if (VERBOSE) {
          System.out.println("TEST: got expected exc:");
          e.printStackTrace(System.out);
        }
        // Make sure exc message includes a path=
        assertTrue("got exc message: " + e.getMessage(), e.getMessage().indexOf("path=\"") != -1);
      } finally {
        // we should fail to open IW, and so it should be null when we get here.
        // However, if the test fails (i.e., IW did not fail on open), we need
        // to close IW. However, if merges are run, IW may throw
        // IndexFormatTooOldException, and we don't want to mask the fail()
        // above, so close without waiting for merges.
        if (writer != null) {
          writer.close(false);
        }
        writer = null;
      }
     
      ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
      CheckIndex checker = new CheckIndex(dir);
      checker.setInfoStream(new PrintStream(bos, false, "UTF-8"));
      CheckIndex.Status indexStatus = checker.checkIndex();
      assertFalse(indexStatus.clean);
      assertTrue(bos.toString("UTF-8").contains(IndexFormatTooOldException.class.getName()));

      dir.close();
      _TestUtil.rmDir(oldIndxeDir);
    }
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

  // Simulate a corrupt index by removing last byte of
  // latest segments file and make sure we get an
  // IOException trying to open the index:
  public void testSimulatedCorruptIndex1() throws IOException {
      BaseDirectoryWrapper dir = newDirectory();
      dir.setCheckIndexOnClose(false); // we are corrupting it!

      IndexWriter writer = null;

      writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));

      // add 100 documents
      for (int i = 0; i < 100; i++) {
          addDoc(writer);
      }

      // close
      writer.close();

      long gen = SegmentInfos.getLastCommitGeneration(dir);
      assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

      String fileNameIn = SegmentInfos.getLastCommitSegmentsFileName(dir);
      String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                 "",
                                                                 1+gen);
      IndexInput in = dir.openInput(fileNameIn, newIOContext(random()));
      IndexOutput out = dir.createOutput(fileNameOut, newIOContext(random()));
      long length = in.length();
      for(int i=0;i<length-1;i++) {
        out.writeByte(in.readByte());
      }
      in.close();
      out.close();
      dir.deleteFile(fileNameIn);

      IndexReader reader = null;
      try {
        reader = DirectoryReader.open(dir);
        fail("reader did not hit IOException on opening a corrupt index");
      } catch (Exception e) {
      }
      if (reader != null) {
        reader.close();
      }
      dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

  // Simulate a corrupt index by removing one of the cfs
  // files and make sure we get an IOException trying to
  // open the index:
  public void testSimulatedCorruptIndex2() throws IOException {
    BaseDirectoryWrapper dir = newDirectory();
    dir.setCheckIndexOnClose(false); // we are corrupting it!
    IndexWriter writer = null;

    writer  = new IndexWriter(
                              dir,
                              newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
                              setMergePolicy(newLogMergePolicy(true))
                              );
    LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
    // Force creation of CFS:
    lmp.setNoCFSRatio(1.0);
    lmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);

    // add 100 documents
    for (int i = 0; i < 100; i++) {
      addDoc(writer);
    }

    // close
    writer.close();

    long gen = SegmentInfos.getLastCommitGeneration(dir);
    assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

    String[] files = dir.listAll();
    boolean corrupted = false;
    for(int i=0;i<files.length;i++) {
      if (files[i].endsWith(".cfs")) {
        dir.deleteFile(files[i]);
        corrupted = true;
        break;
      }
    }
    assertTrue("failed to find cfs file to remove", corrupted);

    IndexReader reader = null;
    try {
      reader = DirectoryReader.open(dir);
      fail("reader did not hit IOException on opening a corrupt index");
    } catch (Exception e) {
    }
    if (reader != null) {
      reader.close();
    }
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

      if (VERBOSE) {
        System.out.println("TEST: index " + unsupportedNames[i]);
      }
      File oldIndxeDir = _TestUtil.getTempDir(unsupportedNames[i]);
      _TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
      BaseDirectoryWrapper dir = newFSDirectory(oldIndxeDir);
      // don't checkindex, these are intentionally not supported
      dir.setCheckIndexOnClose(false);

      IndexReader reader = null;
      IndexWriter writer = null;
      try {
        reader = DirectoryReader.open(dir);
        fail("DirectoryReader.open should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
      } finally {
        if (reader != null) reader.close();
        reader = null;
      }

      try {
        writer = new IndexWriter(dir, newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random())));
        fail("IndexWriter creation should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
        if (VERBOSE) {
          System.out.println("TEST: got expected exc:");
          e.printStackTrace(System.out);
        }
        // Make sure exc message includes a path=
        assertTrue("got exc message: " + e.getMessage(), e.getMessage().indexOf("path=\"") != -1);
      } finally {
        // we should fail to open IW, and so it should be null when we get here.
        // However, if the test fails (i.e., IW did not fail on open), we need
        // to close IW. However, if merges are run, IW may throw
        // IndexFormatTooOldException, and we don't want to mask the fail()
        // above, so close without waiting for merges.
        if (writer != null) {
          writer.close(false);
        }
        writer = null;
      }
     
      ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
      CheckIndex checker = new CheckIndex(dir);
      checker.setInfoStream(new PrintStream(bos, false, "UTF-8"));
      CheckIndex.Status indexStatus = checker.checkIndex();
      assertFalse(indexStatus.clean);
      assertTrue(bos.toString("UTF-8").contains(IndexFormatTooOldException.class.getName()));

      dir.close();
      _TestUtil.rmDir(oldIndxeDir);
    }
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

    final int NUM_DOCS = atLeast(173);
    if (VERBOSE) {
      System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
    }

    BaseDirectoryWrapper dir = newDirectory();
    dir.setCheckIndexOnClose(false); // we use a custom codec provider
    IndexWriter w = new IndexWriter(
        dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
        setCodec(new CustomPerFieldCodec()).
            setMergePolicy(newLogMergePolicy(3))
    );
    Document doc = new Document();
    // uses default codec:
    doc.add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO));
    // uses pulsing codec:
    Field field2 = newTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO);
    doc.add(field2);
   
    Field idField = newStringField("id", "", Field.Store.NO);

    doc.add(idField);
    for(int i=0;i<NUM_DOCS;i++) {
      idField.setStringValue(""+i);
      w.addDocument(doc);
      if ((i+1)%10 == 0) {
        w.commit();
      }
    }
    if (VERBOSE) {
      System.out.println("TEST: now delete id=77");
    }
    w.deleteDocuments(new Term("id", "77"));

    IndexReader r = DirectoryReader.open(w, true);
   
    assertEquals(NUM_DOCS-1, r.numDocs());
    IndexSearcher s = newSearcher(r);
    assertEquals(NUM_DOCS-1, s.search(new TermQuery(new Term("field1", "standard")), 1).totalHits);
    assertEquals(NUM_DOCS-1, s.search(new TermQuery(new Term("field2", "pulsing")), 1).totalHits);
    r.close();

    if (VERBOSE) {
      System.out.println("\nTEST: now delete 2nd doc");
    }
    w.deleteDocuments(new Term("id", "44"));

    if (VERBOSE) {
      System.out.println("\nTEST: now force merge");
    }
    w.forceMerge(1);
    if (VERBOSE) {
      System.out.println("\nTEST: now open reader");
    }
    r = DirectoryReader.open(w, true);
    assertEquals(NUM_DOCS-2, r.maxDoc());
    assertEquals(NUM_DOCS-2, r.numDocs());
    s = newSearcher(r);
    assertEquals(NUM_DOCS-2, s.search(new TermQuery(new Term("field1", "standard")), 1).totalHits);
    assertEquals(NUM_DOCS-2, s.search(new TermQuery(new Term("field2", "pulsing")), 1).totalHits);
    assertEquals(1, s.search(new TermQuery(new Term("id", "76")), 1).totalHits);
    assertEquals(0, s.search(new TermQuery(new Term("id", "77")), 1).totalHits);
    assertEquals(0, s.search(new TermQuery(new Term("id", "44")), 1).totalHits);

    if (VERBOSE) {
      System.out.println("\nTEST: now close NRT reader");
    }
    r.close();

    w.close();

    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

@Monster("You must increase heap to > 2 G to run this")
public class Test2BPagedBytes extends LuceneTestCase {

  public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("test2BPagedBytes"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    PagedBytes pb = new PagedBytes(15);
    IndexOutput dataOutput = dir.createOutput("foo", IOContext.DEFAULT);
    long netBytes = 0;
    long seed = random().nextLong();
    long lastFP = 0;
    Random r2 = new Random(seed);
    while(netBytes < 1.1*Integer.MAX_VALUE) {
      int numBytes = TestUtil.nextInt(r2, 1, 32768);
      byte[] bytes = new byte[numBytes];
      r2.nextBytes(bytes);
      dataOutput.writeBytes(bytes, bytes.length);
      long fp = dataOutput.getFilePointer();
      assert fp == lastFP + numBytes;
      lastFP = fp;
      netBytes += numBytes;
    }
    dataOutput.close();
    IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
    pb.copy(input, input.length());
    input.close();
    PagedBytes.Reader reader = pb.freeze(true);

    r2 = new Random(seed);
    netBytes = 0;
    while(netBytes < 1.1*Integer.MAX_VALUE) {
      int numBytes = TestUtil.nextInt(r2, 1, 32768);
      byte[] bytes = new byte[numBytes];
      r2.nextBytes(bytes);
      BytesRef expected = new BytesRef(bytes);

      BytesRef actual = new BytesRef();
      reader.fillSlice(actual, netBytes, numBytes);
      assertEquals(expected, actual);

      netBytes += numBytes;
    }
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

  // copies into PagedBytes and verifies with
  // PagedBytes.Reader:
  public void testDataInputOutput() throws Exception {
    Random random = random();
    for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
      BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
      if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
      }
      final int blockBits = TestUtil.nextInt(random, 1, 20);
      final int blockSize = 1 << blockBits;
      final PagedBytes p = new PagedBytes(blockBits);
      final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
      final int numBytes = TestUtil.nextInt(random(), 2, 10000000);

      final byte[] answer = new byte[numBytes];
      random().nextBytes(answer);
      int written = 0;
      while(written < numBytes) {
        if (random().nextInt(10) == 7) {
          out.writeByte(answer[written++]);
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - written);
          out.writeBytes(answer, written, chunk);
          written += chunk;
        }
      }
     
      out.close();
      final IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
      final DataInput in = input.clone();
     
      p.copy(input, input.length());
      final PagedBytes.Reader reader = p.freeze(random.nextBoolean());

      final byte[] verify = new byte[numBytes];
      int read = 0;
      while(read < numBytes) {
        if (random().nextInt(10) == 7) {
          verify[read++] = in.readByte();
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - read);
          in.readBytes(verify, read, chunk);
          read += chunk;
        }
      }
      assertTrue(Arrays.equals(answer, verify));

      final BytesRef slice = new BytesRef();
      for(int iter2=0;iter2<100;iter2++) {
        final int pos = random.nextInt(numBytes-1);
        final int len = random.nextInt(Math.min(blockSize+1, numBytes - pos));
        reader.fillSlice(slice, pos, len);
        for(int byteUpto=0;byteUpto<len;byteUpto++) {
          assertEquals(answer[pos + byteUpto], slice.bytes[slice.offset + byteUpto]);
        }
      }
      input.close();
      dir.close();
    }
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

    }
  }

  @Ignore // memory hole
  public void testOverflow() throws IOException {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    final int blockBits = TestUtil.nextInt(random(), 14, 28);
    final int blockSize = 1 << blockBits;
    byte[] arr = new byte[TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
    for (int i = 0; i < arr.length; ++i) {
      arr[i] = (byte) i;
    }
    final long numBytes = (1L << 31) + TestUtil.nextInt(random(), 1, blockSize * 3);
    final PagedBytes p = new PagedBytes(blockBits);
    final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
    for (long i = 0; i < numBytes; ) {
      assertEquals(i, out.getFilePointer());
      final int len = (int) Math.min(arr.length, numBytes - i);
      out.writeBytes(arr, len);
      i += len;
    }
    assertEquals(numBytes, out.getFilePointer());
    out.close();
    final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
    p.copy(in, numBytes);
    final PagedBytes.Reader reader = p.freeze(random().nextBoolean());

    for (long offset : new long[] {0L, Integer.MAX_VALUE, numBytes - 1,
        TestUtil.nextLong(random(), 1, numBytes - 2)}) {
      BytesRef b = new BytesRef();
      reader.fillSlice(b, offset, 1);
      assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
    }
    in.close();
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

        fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES);
        clazz = CommandLineUtil.loadFSDirectoryClass(fsdirClass);
      }

      Directory fsdir = newFSDirectoryImpl(clazz, f);
      BaseDirectoryWrapper wrapped = wrapDirectory(random(), fsdir, bare);
      if (lf != null) {
        wrapped.setLockFactory(lf);
      }
      return wrapped;
    } catch (Exception e) {
      Rethrow.rethrow(e);
      throw null; // dummy to prevent compiler failure
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

      directory =  rateLimitedDirectoryWrapper;
     
    }

    if (bare) {
      BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
      closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
      return base;
    } else {
      MockDirectoryWrapper mock = new MockDirectoryWrapper(random, directory);
     
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.