Examples of BaseDirectoryWrapper


Examples of org.apache.lucene.store.BaseDirectoryWrapper

@Ignore("You must increase heap to > 2 G to run this")
public class Test2BPagedBytes extends LuceneTestCase {

  public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("test2BPagedBytes"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    PagedBytes pb = new PagedBytes(15);
    IndexOutput dataOutput = dir.createOutput("foo", IOContext.DEFAULT);
    long netBytes = 0;
    long seed = random().nextLong();
    long lastFP = 0;
    Random r2 = new Random(seed);
    while(netBytes < 1.1*Integer.MAX_VALUE) {
      int numBytes = _TestUtil.nextInt(r2, 1, 32768);
      byte[] bytes = new byte[numBytes];
      r2.nextBytes(bytes);
      dataOutput.writeBytes(bytes, bytes.length);
      long fp = dataOutput.getFilePointer();
      assert fp == lastFP + numBytes;
      lastFP = fp;
      netBytes += numBytes;
    }
    dataOutput.close();
    IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
    pb.copy(input, input.length());
    input.close();
    PagedBytes.Reader reader = pb.freeze(true);

    r2 = new Random(seed);
    netBytes = 0;
    while(netBytes < 1.1*Integer.MAX_VALUE) {
      int numBytes = _TestUtil.nextInt(r2, 1, 32768);
      byte[] bytes = new byte[numBytes];
      r2.nextBytes(bytes);
      BytesRef expected = new BytesRef(bytes);

      BytesRef actual = new BytesRef();
      reader.fillSlice(actual, netBytes, numBytes);
      assertEquals(expected, actual);

      netBytes += numBytes;
    }
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

  // copies into PagedBytes and verifies with
  // PagedBytes.Reader:
  public void testDataInputOutput() throws Exception {
    Random random = random();
    for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
      BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("testOverflow"));
      if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
      }
      final int blockBits = _TestUtil.nextInt(random, 1, 20);
      final int blockSize = 1 << blockBits;
      final PagedBytes p = new PagedBytes(blockBits);
      final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
      final int numBytes = _TestUtil.nextInt(random(), 2, 10000000);

      final byte[] answer = new byte[numBytes];
      random().nextBytes(answer);
      int written = 0;
      while(written < numBytes) {
        if (random().nextInt(10) == 7) {
          out.writeByte(answer[written++]);
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - written);
          out.writeBytes(answer, written, chunk);
          written += chunk;
        }
      }
     
      out.close();
      final IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
      final DataInput in = input.clone();
     
      p.copy(input, input.length());
      final PagedBytes.Reader reader = p.freeze(random.nextBoolean());

      final byte[] verify = new byte[numBytes];
      int read = 0;
      while(read < numBytes) {
        if (random().nextInt(10) == 7) {
          verify[read++] = in.readByte();
        } else {
          int chunk = Math.min(random().nextInt(1000), numBytes - read);
          in.readBytes(verify, read, chunk);
          read += chunk;
        }
      }
      assertTrue(Arrays.equals(answer, verify));

      final BytesRef slice = new BytesRef();
      for(int iter2=0;iter2<100;iter2++) {
        final int pos = random.nextInt(numBytes-1);
        final int len = random.nextInt(Math.min(blockSize+1, numBytes - pos));
        reader.fillSlice(slice, pos, len);
        for(int byteUpto=0;byteUpto<len;byteUpto++) {
          assertEquals(answer[pos + byteUpto], slice.bytes[slice.offset + byteUpto]);
        }
      }
      input.close();
      dir.close();
    }
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

    }
  }

  @Ignore // memory hole
  public void testOverflow() throws IOException {
    BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("testOverflow"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    final int blockBits = _TestUtil.nextInt(random(), 14, 28);
    final int blockSize = 1 << blockBits;
    byte[] arr = new byte[_TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
    for (int i = 0; i < arr.length; ++i) {
      arr[i] = (byte) i;
    }
    final long numBytes = (1L << 31) + _TestUtil.nextInt(random(), 1, blockSize * 3);
    final PagedBytes p = new PagedBytes(blockBits);
    final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
    for (long i = 0; i < numBytes; ) {
      assertEquals(i, out.getFilePointer());
      final int len = (int) Math.min(arr.length, numBytes - i);
      out.writeBytes(arr, len);
      i += len;
    }
    assertEquals(numBytes, out.getFilePointer());
    out.close();
    final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
    p.copy(in, numBytes);
    final PagedBytes.Reader reader = p.freeze(random().nextBoolean());

    for (long offset : new long[] {0L, Integer.MAX_VALUE, numBytes - 1,
        _TestUtil.nextLong(random(), 1, numBytes - 2)}) {
      BytesRef b = new BytesRef();
      reader.fillSlice(b, offset, 1);
      assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
    }
    in.close();
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

 
  /** tests reuse with Pulsing1(Pulsing2(Standard)) */
  public void testNestedPulsing() throws Exception {
    // we always run this test with pulsing codec.
    Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat());
    BaseDirectoryWrapper dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
    Document doc = new Document();
    doc.add(new TextField("foo", "a b b c c c d e f g g g h i i j j k l l m m m", Field.Store.NO));
    // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm')
    // this is because we only track the 'last' enum we reused (not all).
    // but this seems 'good enough' for now.
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
   
    AtomicReader segment = getOnlySegmentReader(ir);
    DocsEnum reuse = null;
    Map<DocsEnum,Boolean> allEnums = new IdentityHashMap<DocsEnum,Boolean>();
    TermsEnum te = segment.terms("foo").iterator(null);
    while (te.next() != null) {
      reuse = te.docs(null, reuse, DocsEnum.FLAG_NONE);
      allEnums.put(reuse, true);
    }
   
    assertEquals(4, allEnums.size());
   
    allEnums.clear();
    DocsAndPositionsEnum posReuse = null;
    te = segment.terms("foo").iterator(null);
    while (te.next() != null) {
      posReuse = te.docsAndPositions(null, posReuse);
      allEnums.put(posReuse, true);
    }
   
    assertEquals(4, allEnums.size());
   
    ir.close();
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

  public void test10kPulsed() throws Exception {
    // we always run this test with pulsing codec.
    Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(1));
   
    File f = _TestUtil.getTempDir("10kpulsed");
    BaseDirectoryWrapper dir = newFSDirectory(f);
    dir.setCheckIndexOnClose(false); // we do this ourselves explicitly
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
   
    Document document = new Document();
    FieldType ft = new FieldType(TextField.TYPE_STORED);
   
    switch(_TestUtil.nextInt(random(), 0, 2)) {
      case 0: ft.setIndexOptions(IndexOptions.DOCS_ONLY); break;
      case 1: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); break;
      default: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); break;
    }

    Field field = newField("field", "", ft);
    document.add(field);
   
    NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ROOT));

    for (int i = 0; i < 10050; i++) {
      field.setStringValue(df.format(i));
      iw.addDocument(document);
    }
   
    IndexReader ir = iw.getReader();
    iw.close();

    TermsEnum te = MultiFields.getTerms(ir, "field").iterator(null);
    DocsEnum de = null;
   
    for (int i = 0; i < 10050; i++) {
      String expected = df.format(i);
      assertEquals(expected, te.next().utf8ToString());
      de = _TestUtil.docs(random(), te, null, de, DocsEnum.FLAG_NONE);
      assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
      assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
    }
    ir.close();

    _TestUtil.checkIndex(dir);
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

    // we always run this test with pulsing codec.
    int freqCutoff = _TestUtil.nextInt(random(), 1, 10);
    Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(freqCutoff));
   
    File f = _TestUtil.getTempDir("10knotpulsed");
    BaseDirectoryWrapper dir = newFSDirectory(f);
    dir.setCheckIndexOnClose(false); // we do this ourselves explicitly
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
   
    Document document = new Document();
    FieldType ft = new FieldType(TextField.TYPE_STORED);
   
    switch(_TestUtil.nextInt(random(), 0, 2)) {
      case 0: ft.setIndexOptions(IndexOptions.DOCS_ONLY); break;
      case 1: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); break;
      default: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); break;
    }

    Field field = newField("field", "", ft);
    document.add(field);
   
    NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ROOT));

    final int freq = freqCutoff + 1;
   
    for (int i = 0; i < 10050; i++) {
      StringBuilder sb = new StringBuilder();
      for (int j = 0; j < freq; j++) {
        sb.append(df.format(i));
        sb.append(' '); // whitespace
      }
      field.setStringValue(sb.toString());
      iw.addDocument(document);
    }
   
    IndexReader ir = iw.getReader();
    iw.close();

    TermsEnum te = MultiFields.getTerms(ir, "field").iterator(null);
    DocsEnum de = null;
   
    for (int i = 0; i < 10050; i++) {
      String expected = df.format(i);
      assertEquals(expected, te.next().utf8ToString());
      de = _TestUtil.docs(random(), te, null, de, DocsEnum.FLAG_NONE);
      assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
      assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
    }
    ir.close();

    _TestUtil.checkIndex(dir);
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
public class Test2BPostings extends LuceneTestCase {

  @Nightly
  public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }

    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
        .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
        .setRAMBufferSizeMB(256.0)
        .setMergeScheduler(new ConcurrentMergeScheduler())
        .setMergePolicy(newLogMergePolicy(false, 10))
        .setOpenMode(IndexWriterConfig.OpenMode.CREATE);
   
    IndexWriter w = new IndexWriter(dir, iwc);

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
     // 1 petabyte:
     ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
    }

    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setOmitNorms(true);
    ft.setIndexOptions(IndexOptions.DOCS_ONLY);
    Field field = new Field("field", new MyTokenStream(), ft);
    doc.add(field);
   
    final int numDocs = (Integer.MAX_VALUE / 26) + 1;
    for (int i = 0; i < numDocs; i++) {
      w.addDocument(doc);
      if (VERBOSE && i % 100000 == 0) {
        System.out.println(i + " of " + numDocs + "...");
      }
    }
    w.forceMerge(1);
    w.close();
    dir.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

      if (VERBOSE) {
        System.out.println("TEST: index " + unsupportedNames[i]);
      }
      File oldIndxeDir = _TestUtil.getTempDir(unsupportedNames[i]);
      _TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
      BaseDirectoryWrapper dir = newFSDirectory(oldIndxeDir);
      // don't checkindex, these are intentionally not supported
      dir.setCheckIndexOnClose(false);

      IndexReader reader = null;
      IndexWriter writer = null;
      try {
        reader = DirectoryReader.open(dir);
        fail("DirectoryReader.open should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
      } finally {
        if (reader != null) reader.close();
        reader = null;
      }

      try {
        writer = new IndexWriter(dir, newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random())));
        fail("IndexWriter creation should not pass for "+unsupportedNames[i]);
      } catch (IndexFormatTooOldException e) {
        // pass
        if (VERBOSE) {
          System.out.println("TEST: got expected exc:");
          e.printStackTrace(System.out);
        }
        // Make sure exc message includes a path=
        assertTrue("got exc message: " + e.getMessage(), e.getMessage().indexOf("path=\"") != -1);
      } finally {
        // we should fail to open IW, and so it should be null when we get here.
        // However, if the test fails (i.e., IW did not fail on open), we need
        // to close IW. However, if merges are run, IW may throw
        // IndexFormatTooOldException, and we don't want to mask the fail()
        // above, so close without waiting for merges.
        if (writer != null) {
          writer.close(false);
        }
        writer = null;
      }
     
      ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
      CheckIndex checker = new CheckIndex(dir);
      checker.setInfoStream(new PrintStream(bos, false, "UTF-8"));
      CheckIndex.Status indexStatus = checker.checkIndex();
      assertFalse(indexStatus.clean);
      assertTrue(bos.toString("UTF-8").contains(IndexFormatTooOldException.class.getName()));

      dir.close();
      _TestUtil.rmDir(oldIndxeDir);
    }
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

    }
  }

  // LUCENE-4147
  public void testRollbackAndCommitWithThreads() throws Exception {
    final BaseDirectoryWrapper d = newDirectory();
    if (d instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)d).setPreventDoubleWrite(false);
    }

    final int threadCount = _TestUtil.nextInt(random(), 2, 6);

    final AtomicReference<IndexWriter> writerRef = new AtomicReference<IndexWriter>();
    writerRef.set(new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))));
    final LineFileDocs docs = new LineFileDocs(random());
    final Thread[] threads = new Thread[threadCount];
    final int iters = atLeast(100);
    final AtomicBoolean failed = new AtomicBoolean();
    final Lock rollbackLock = new ReentrantLock();
    final Lock commitLock = new ReentrantLock();
    for(int threadID=0;threadID<threadCount;threadID++) {
      threads[threadID] = new Thread() {
          @Override
          public void run() {
            for(int iter=0;iter<iters && !failed.get();iter++) {
              //final int x = random().nextInt(5);
              final int x = random().nextInt(3);
              try {
                switch(x) {
                case 0:
                  rollbackLock.lock();
                  if (VERBOSE) {
                    System.out.println("\nTEST: " + Thread.currentThread().getName() + ": now rollback");
                  }
                  try {
                    writerRef.get().rollback();
                    if (VERBOSE) {
                      System.out.println("TEST: " + Thread.currentThread().getName() + ": rollback done; now open new writer");
                    }
                    writerRef.set(new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))));
                  } finally {
                    rollbackLock.unlock();
                  }
                  break;
                case 1:
                  commitLock.lock();
                  if (VERBOSE) {
                    System.out.println("\nTEST: " + Thread.currentThread().getName() + ": now commit");
                  }
                  try {
                    if (random().nextBoolean()) {
                      writerRef.get().prepareCommit();
                    }
                    writerRef.get().commit();
                  } catch (AlreadyClosedException ace) {
                    // ok
                  } catch (NullPointerException npe) {
                    // ok
                  } finally {
                    commitLock.unlock();
                  }
                  break;
                case 2:
                  if (VERBOSE) {
                    System.out.println("\nTEST: " + Thread.currentThread().getName() + ": now add");
                  }
                  try {
                    writerRef.get().addDocument(docs.nextDoc());
                  } catch (AlreadyClosedException ace) {
                    // ok
                  } catch (NullPointerException npe) {
                    // ok
                  } catch (AssertionError ae) {
                    // ok
                  }
                  break;
                }
              } catch (Throwable t) {
                failed.set(true);
                throw new RuntimeException(t);
              }
            }
          }
        };
      threads[threadID].start();
    }

    for(int threadID=0;threadID<threadCount;threadID++) {
      threads[threadID].join();
    }

    assertTrue(!failed.get());
    writerRef.get().close();
    d.close();
  }
View Full Code Here

Examples of org.apache.lucene.store.BaseDirectoryWrapper

public class TestNeverDelete extends LuceneTestCase {

  public void testIndexing() throws Exception {
    final File tmpDir = _TestUtil.getTempDir("TestNeverDelete");
    final BaseDirectoryWrapper d = newFSDirectory(tmpDir);

    // We want to "see" files removed if Lucene removed
    // them.  This is still worth running on Windows since
    // some files the IR opens and closes.
    if (d instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper)d).setNoDeleteOpenFile(false);
    }
    final RandomIndexWriter w = new RandomIndexWriter(random(),
                                                      d,
                                                      newIndexWriterConfig(TEST_VERSION_CURRENT,
                                                                           new MockAnalyzer(random()))
                                                      .setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
    w.w.getConfig().setMaxBufferedDocs(_TestUtil.nextInt(random(), 5, 30));

    w.commit();
    Thread[] indexThreads = new Thread[random().nextInt(4)];
    final long stopTime = System.currentTimeMillis() + atLeast(1000);
    for (int x=0; x < indexThreads.length; x++) {
      indexThreads[x] = new Thread() {
          @Override
          public void run() {
            try {
              int docCount = 0;
              while (System.currentTimeMillis() < stopTime) {
                final Document doc = new Document();
                doc.add(newStringField("dc", ""+docCount, Field.Store.YES));
                doc.add(newTextField("field", "here is some text", Field.Store.YES));
                w.addDocument(doc);

                if (docCount % 13 == 0) {
                  w.commit();
                }
                docCount++;
              }
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
        };
      indexThreads[x].setName("Thread " + x);
      indexThreads[x].start();
    }

    final Set<String> allFiles = new HashSet<String>();

    DirectoryReader r = DirectoryReader.open(d);
    while(System.currentTimeMillis() < stopTime) {
      final IndexCommit ic = r.getIndexCommit();
      if (VERBOSE) {
        System.out.println("TEST: check files: " + ic.getFileNames());
      }
      allFiles.addAll(ic.getFileNames());
      // Make sure no old files were removed
      for(String fileName : allFiles) {
        assertTrue("file " + fileName + " does not exist", d.fileExists(fileName));
      }
      DirectoryReader r2 = DirectoryReader.openIfChanged(r);
      if (r2 != null) {
        r.close();
        r = r2;
      }
      Thread.sleep(1);
    }
    r.close();

    for(Thread t : indexThreads) {
      t.join();
    }
    w.close();
    d.close();

    _TestUtil.rmDir(tmpDir);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.