Examples of TieredMergePolicy

org.apache.lucene.index.TieredMergePolicy
Merges segments of approximately equal size, subject to an allowed number of segments per tier. This is similar to {@link LogByteSizeMergePolicy}, except this merge policy is able to merge non-adjacent segment, and separates how many segments are merged at once ( {@link #setMaxMergeAtOnce}) from how many segments are allowed per tier ( {@link #setSegmentsPerTier}). This merge policy also does not over-merge (ie, cascade merges).
For normal merging, this policy first computes a "budget" of how many segments are allowed by be in the index. If the index is over-budget, then the policy sorts segments by decresing size (pro-rating by percent deletes), and then finds the least-cost merge. Merge cost is measured by a combination of the "skew" of the merge (size of largest seg divided by smallest seg), total merge size and pct deletes reclaimed, so that merges with lower skew, smaller size and those reclaiming more deletes, are favored.
If a merge will produce a segment that's larger than {@link #setMaxMergedSegmentMB}, then the policy will merge fewer segments (down to 1 at once, if that one has deletions) to keep the segment size under budget. NOTE: this policy freely merges non-adjacent segments; if this is a problem, use {@link LogMergePolicy}.
NOTE: This policy always merges by byte size of the segments, always pro-rates by percent deletes, and does not apply any maximum segment size during optimize (unlike {@link LogByteSizeMergePolicy}. @lucene.experimental

Examples of org.apache.lucene.index.TieredMergePolicy

      if(iwConf.getMergePolicy() instanceof LogMergePolicy) {
        LogMergePolicy logMergePolicy = (LogMergePolicy) iwConf.getMergePolicy();
        logMergePolicy.setUseCompoundFile(isCompound);
        logMergePolicy.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR));
      } else if(iwConf.getMergePolicy() instanceof TieredMergePolicy) {
        TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) iwConf.getMergePolicy();
        tieredMergePolicy.setUseCompoundFile(isCompound);
      }
    }
    final double ramBuffer = config.get("ram.flush.mb",OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
    final int maxBuffered = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED);
    if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH) {

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    }


    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

      mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
    }
  }


  public static TieredMergePolicy newTieredMergePolicy(Random r) {
    TieredMergePolicy tmp = new TieredMergePolicy();
    if (rarely(r)) {
      tmp.setMaxMergeAtOnce(TestUtil.nextInt(r, 2, 9));
      tmp.setMaxMergeAtOnceExplicit(TestUtil.nextInt(r, 2, 9));
    } else {
      tmp.setMaxMergeAtOnce(TestUtil.nextInt(r, 10, 50));
      tmp.setMaxMergeAtOnceExplicit(TestUtil.nextInt(r, 10, 50));
    }
    if (rarely(r)) {
      tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
    } else {
      tmp.setMaxMergedSegmentMB(r.nextDouble() * 100);
    }
    tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
    tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
    if (rarely(r)) {
      tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20));
    } else {
      tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50));
    }
    configureRandom(r, tmp);
    tmp.setReclaimDeletesWeight(r.nextDouble()*4);
    return tmp;
  }

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

          logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9));
        } else {
          logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50));
        }
      } else if (mp instanceof TieredMergePolicy) {
        TieredMergePolicy tmp = (TieredMergePolicy) mp;
        if (rarely(r)) {
          tmp.setMaxMergeAtOnce(TestUtil.nextInt(r, 2, 9));
          tmp.setMaxMergeAtOnceExplicit(TestUtil.nextInt(r, 2, 9));
        } else {
          tmp.setMaxMergeAtOnce(TestUtil.nextInt(r, 10, 50));
          tmp.setMaxMergeAtOnceExplicit(TestUtil.nextInt(r, 10, 50));
        }
        if (rarely(r)) {
          tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
        } else {
          tmp.setMaxMergedSegmentMB(r.nextDouble() * 100);
        }
        tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
        tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
        if (rarely(r)) {
          tmp.setSegmentsPerTier(TestUtil.nextInt(r, 2, 20));
        } else {
          tmp.setSegmentsPerTier(TestUtil.nextInt(r, 10, 50));
        }
        configureRandom(r, tmp);
        tmp.setReclaimDeletesWeight(r.nextDouble()*4);
      }
      didChange = true;
    }
    if (VERBOSE && didChange) {
      System.out.println("NOTE: LuceneTestCase: randomly changed IWC's live settings to:\n" + c);

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

  static MergePolicy newSortingMergePolicy(Sort sort) {
    // usually create a MP with a low merge factor so that many merges happen
    MergePolicy mp;
    int thingToDo = random().nextInt(3);
    if (thingToDo == 0) {
      TieredMergePolicy tmp = newTieredMergePolicy(random());
      final int numSegs = TestUtil.nextInt(random(), 3, 5);
      tmp.setSegmentsPerTier(numSegs);
      tmp.setMaxMergeAtOnce(TestUtil.nextInt(random(), 2, numSegs));
      mp = tmp;
    } else if (thingToDo == 1) {
      LogMergePolicy lmp = newLogMergePolicy(random());
      lmp.setMergeFactor(TestUtil.nextInt(random(), 3, 5));
      mp = lmp;

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy


  static MergePolicy newSortingMergePolicy(Sorter sorter) {
    // create a MP with a low merge factor so that many merges happen
    MergePolicy mp;
    if (random().nextBoolean()) {
      TieredMergePolicy tmp = newTieredMergePolicy(random());
      final int numSegs = _TestUtil.nextInt(random(), 3, 5);
      tmp.setSegmentsPerTier(numSegs);
      tmp.setMaxMergeAtOnce(_TestUtil.nextInt(random(), 2, numSegs));
      mp = tmp;
    } else {
      LogMergePolicy lmp = newLogMergePolicy(random());
      lmp.setMergeFactor(_TestUtil.nextInt(random(), 3, 5));
      mp = lmp;

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setNoCFSRatio(1.0);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setNoCFSRatio(1.0);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      // wtf... shouldnt it be even lower since its 1 by default?!?!
      ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);

View Full Code Here

Examples of org.apache.lucene.index.TieredMergePolicy

    if (mp instanceof LogMergePolicy) {
      LogMergePolicy lmp = (LogMergePolicy) mp;
      lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
      lmp.setUseCompoundFile(true);
    } else if (mp instanceof TieredMergePolicy) {
      TieredMergePolicy tmp = (TieredMergePolicy) mp;
      tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
      tmp.setUseCompoundFile(true);
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
      ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
      ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.