final int TERMS_PER_DOC = TestUtil.nextInt(random(), 100000, 1000000);
List<BytesRef> savedTerms = null;
BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BTerms"));
//MockDirectoryWrapper dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
}
dir.setCheckIndexOnClose(false); // don't double-checkindex
if (true) {
IndexWriter w = new IndexWriter(dir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setRAMBufferSizeMB(256.0)
.setMergeScheduler(new ConcurrentMergeScheduler())
.setMergePolicy(newLogMergePolicy(false, 10))
.setOpenMode(IndexWriterConfig.OpenMode.CREATE));
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
// 1 petabyte:
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
}
Document doc = new Document();
final MyTokenStream ts = new MyTokenStream(random(), TERMS_PER_DOC);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setIndexOptions(IndexOptions.DOCS_ONLY);
customType.setOmitNorms(true);
Field field = new Field("field", ts, customType);
doc.add(field);
//w.setInfoStream(System.out);
final int numDocs = (int) (TERM_COUNT/TERMS_PER_DOC);
System.out.println("TERMS_PER_DOC=" + TERMS_PER_DOC);
System.out.println("numDocs=" + numDocs);
for(int i=0;i<numDocs;i++) {
final long t0 = System.currentTimeMillis();
w.addDocument(doc);
System.out.println(i + " of " + numDocs + " " + (System.currentTimeMillis()-t0) + " msec");
}
savedTerms = ts.savedTerms;
System.out.println("TEST: full merge");
w.forceMerge(1);
System.out.println("TEST: close writer");
w.close();
}
System.out.println("TEST: open reader");
final IndexReader r = DirectoryReader.open(dir);
if (savedTerms == null) {
savedTerms = findTerms(r);
}
final int numSavedTerms = savedTerms.size();
final List<BytesRef> bigOrdTerms = new ArrayList<>(savedTerms.subList(numSavedTerms-10, numSavedTerms));
System.out.println("TEST: test big ord terms...");
testSavedTerms(r, bigOrdTerms);
System.out.println("TEST: test all saved terms...");
testSavedTerms(r, savedTerms);
r.close();
System.out.println("TEST: now CheckIndex...");
CheckIndex.Status status = TestUtil.checkIndex(dir);
final long tc = status.segmentInfos.get(0).termIndexStatus.termCount;
assertTrue("count " + tc + " is not > " + Integer.MAX_VALUE, tc > Integer.MAX_VALUE);
dir.close();
System.out.println("TEST: done!");
}