Examples of org.apache.lucene.index.TieredMergePolicy

org.apache.lucene.index.TieredMergePolicy
Merges segments of approximately equal size, subject to an allowed number of segments per tier. This is similar to {@link LogByteSizeMergePolicy}, except this merge policy is able to merge non-adjacent segment, and separates how many segments are merged at once ( {@link #setMaxMergeAtOnce}) from how many segments are allowed per tier ( {@link #setSegmentsPerTier}). This merge policy also does not over-merge (ie, cascade merges).
For normal merging, this policy first computes a "budget" of how many segments are allowed by be in the index. If the index is over-budget, then the policy sorts segments by decresing size (pro-rating by percent deletes), and then finds the least-cost merge. Merge cost is measured by a combination of the "skew" of the merge (size of largest seg divided by smallest seg), total merge size and pct deletes reclaimed, so that merges with lower skew, smaller size and those reclaiming more deletes, are favored.
If a merge will produce a segment that's larger than {@link #setMaxMergedSegmentMB}, then the policy will merge fewer segments (down to 1 at once, if that one has deletions) to keep the segment size under budget. NOTE: this policy freely merges non-adjacent segments; if this is a problem, use {@link LogMergePolicy}.
NOTE: This policy always merges by byte size of the segments, always pro-rates by percent deletes, and does not apply any maximum segment size during optimize (unlike {@link LogByteSizeMergePolicy}. @lucene.experimental

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);

View Full Code Here

    }
    return logmp;
  }


  public static TieredMergePolicy newTieredMergePolicy(Random r) {
    TieredMergePolicy tmp = new TieredMergePolicy();
    if (rarely(r)) {
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4));
      tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4));
    } else {
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50));
      tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50));
    }
    tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
    tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
    tmp.setExpungeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
    tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
    tmp.setUseCompoundFile(r.nextBoolean());
    tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
    tmp.setReclaimDeletesWeight(r.nextDouble()*4);
    return tmp;
  }

View Full Code Here

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    }


    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);

View Full Code Here

      TableContext tableContext = TableContext.create(tableDescriptor);
      _fieldManager = tableContext.getFieldManager();
      Analyzer analyzer = _fieldManager.getAnalyzerForIndex();


      _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
      TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
      mergePolicy.setUseCompoundFile(false);


      _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
      _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);


      if (_indexLocally) {

View Full Code Here


  static MergePolicy newSortingMergePolicy(Sorter sorter) {
    // create a MP with a low merge factor so that many merges happen
    MergePolicy mp;
    if (random().nextBoolean()) {
      TieredMergePolicy tmp = newTieredMergePolicy(random());
      final int numSegs = _TestUtil.nextInt(random(), 3, 5);
      tmp.setSegmentsPerTier(numSegs);
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(random(), 2, numSegs));
      mp = tmp;
    } else {
      LogMergePolicy lmp = newLogMergePolicy(random());
      lmp.setMergeFactor(_TestUtil.nextInt(random(), 3, 5));
      mp = lmp;

View Full Code Here

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);

View Full Code Here

      sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy());
    }
    conf.setIndexDeletionPolicy(sdp);
    conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed));


    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    conf.setMergeScheduler(mergeScheduler.getMergeScheduler());


    DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(directory, gc, closer);
    // This directory allows for warm up by adding tracing ability.
    TraceableDirectory dir = new TraceableDirectory(referenceCounter);

View Full Code Here


  private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
      throws IOException {
    HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);


    Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable, IntWritable>();
    int partition = partitioner.getPartition(new IntWritable(i), null, totalShardCount);
    assertEquals(i, partition);

View Full Code Here

    boolean optimize = _blurTask.getOptimize();


    if (optimize) {
      context.setStatus("Starting Copy-Optimize Phase");
      IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
      TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy();
      policy.setUseCompoundFile(false);
      long s = System.currentTimeMillis();
      IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf);
      writer.addIndexes(reader);
      writer.close();
      long e = System.currentTimeMillis();

View Full Code Here

    nullCheck(_directory);
    nullCheck(_analyzer);
    IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
    config.setSimilarity(new FairSimilarity());
    config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB());
    TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    _writer = new IndexWriter(_directory, config);
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.lucene.index.TieredMergePolicy

org.apache.blur.lucene.warmup.IndexWarmupTest

org.apache.blur.manager.indexserver.DefaultBlurIndexWarmupTest

org.apache.blur.manager.writer.BlurIndexSimpleWriter

org.apache.blur.manager.writer.BlurNRTIndex

org.apache.blur.manager.writer.IndexImporterTest

org.apache.blur.mapreduce.BlurReducer

org.apache.blur.mapreduce.lib.BlurOutputFormat$BlurRecordWriter

org.apache.blur.mapreduce.lib.GenericBlurRecordWriter

org.apache.blur.utils.TableShardCountCollapserTest

org.apache.lucene.benchmark.byTask.tasks.CreateIndexTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.