Package uk.ac.ucl.panda.utility.structure

Examples of uk.ac.ucl.panda.utility.structure.SegmentInfo


  abstract protected long size(SegmentInfo info) throws IOException;

  private boolean isOptimized(SegmentInfos infos, IndexWriter writer, int maxNumSegments, Set segmentsToOptimize) throws IOException {
    final int numSegments = infos.size();
    int numToOptimize = 0;
    SegmentInfo optimizeInfo = null;
    for(int i=0;i<numSegments && numToOptimize <= maxNumSegments;i++) {
      final SegmentInfo info = infos.info(i);
      if (segmentsToOptimize.contains(info)) {
        numToOptimize++;
        optimizeInfo = info;
      }
    }
View Full Code Here


      // Find the newest (rightmost) segment that needs to
      // be optimized (other segments may have been flushed
      // since optimize started):
      int last = infos.size();
      while(last > 0) {
        final SegmentInfo info = infos.info(--last);
        if (segmentsToOptimize.contains(info)) {
          last++;
          break;
        }
      }
View Full Code Here

    final float norm = (float) Math.log(mergeFactor);

    final Directory directory = writer.getDirectory();

    for(int i=0;i<numSegments;i++) {
      final SegmentInfo info = infos.info(i);
      long size = size(info);

      // Refuse to import a segment that's too large
      if (info.docCount > maxMergeDocs && info.dir != directory)
        throw new IllegalArgumentException("Segment is too large (" + info.docCount + " docs vs max docs " + maxMergeDocs + ")");

      if (size >= maxMergeSize && info.dir != directory)
        throw new IllegalArgumentException("Segment is too large (" + size + " vs max size " + maxMergeSize + ")");

      // Floor tiny segments
      if (size < 1)
        size = 1;
      levels[i] = (float) Math.log(size)/norm;
    }

    final float levelFloor;
    if (minMergeSize <= 0)
      levelFloor = (float) 0.0;
    else
      levelFloor = (float) (Math.log(minMergeSize)/norm);

    // Now, we quantize the log values into levels.  The
    // first level is any segment whose log size is within
    // LEVEL_LOG_SPAN of the max size, or, who has such as
    // segment "to the right".  Then, we find the max of all
    // other segments and use that to define the next level
    // segment, etc.

    MergeSpecification spec = null;

    int start = 0;
    while(start < numSegments) {

      // Find max level of all segments not already
      // quantized.
      float maxLevel = levels[start];
      for(int i=1+start;i<numSegments;i++) {
        final float level = levels[i];
        if (level > maxLevel)
          maxLevel = level;
      }

      // Now search backwards for the rightmost segment that
      // falls into this level:
      float levelBottom;
      if (maxLevel < levelFloor)
        // All remaining segments fall into the min level
        levelBottom = -1.0F;
      else {
        levelBottom = (float) (maxLevel - LEVEL_LOG_SPAN);

        // Force a boundary at the level floor
        if (levelBottom < levelFloor && maxLevel >= levelFloor)
          levelBottom = levelFloor;
      }

      int upto = numSegments-1;
      while(upto >= start) {
        if (levels[upto] >= levelBottom) {
          break;
        }
        upto--;
      }
      message("  level " + levelBottom + " to " + maxLevel + ": " + (1+upto-start) + " segments");

      // Finally, record all merges that are viable at this level:
      int end = start + mergeFactor;
      while(end <= 1+upto) {
        boolean anyTooLarge = false;
        for(int i=start;i<end;i++) {
          final SegmentInfo info = infos.info(i);
          anyTooLarge |= (size(info) >= maxMergeSize || info.docCount >= maxMergeDocs);
        }

        if (!anyTooLarge) {
          if (spec == null)
View Full Code Here

    if (!isCommit) {
      // Save files so we can decr on next checkpoint/commit:
      size = segmentInfos.size();
      for(int i=0;i<size;i++) {
        SegmentInfo segmentInfo = segmentInfos.info(i);
        if (segmentInfo.dir == directory) {
          lastFiles.add(segmentInfo.files());
        }
      }
    }
    if (docWriterFiles != null)
      lastFiles.add(docWriterFiles);
View Full Code Here

  }

  void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
    int size = segmentInfos.size();
    for(int i=0;i<size;i++) {
      SegmentInfo segmentInfo = segmentInfos.info(i);
      if (segmentInfo.dir == directory) {
        incRef(segmentInfo.files());
      }
    }

    if (isCommit) {
      // Since this is a commit point, also incref its
View Full Code Here

  }

  void decRef(SegmentInfos segmentInfos) throws IOException {
    final int size = segmentInfos.size();
    for(int i=0;i<size;i++) {
      SegmentInfo segmentInfo = segmentInfos.info(i);
      if (segmentInfo.dir == directory) {
        decRef(segmentInfo.files());
      }
    }
  }
View Full Code Here

      int size = segmentInfos.size();
      files = new ArrayList(size);
      files.add(segmentsFileName);
      gen = segmentInfos.getGeneration();
      for(int i=0;i<size;i++) {
        SegmentInfo segmentInfo = segmentInfos.info(i);
        if (segmentInfo.dir == directory) {
          files.addAll(segmentInfo.files());
        }
      }
    }
View Full Code Here

     
          // Perform the merge
          cfsWriter.close();

          for(int i=0;i<numSegments;i++) {
            SegmentInfo si = segmentInfos.info(i);
            if (si.getDocStoreOffset() != -1 &&
                si.getDocStoreSegment().equals(docStoreSegment))
              si.setDocStoreIsCompoundFile(true);
          }
          checkpoint();
          success = true;
        } finally {
          if (!success) {

            if (infoStream != null)
              message("hit exception building compound file doc store for segment " + docStoreSegment);
           
            // Rollback to no compound file
            for(int i=0;i<numSegments;i++) {
              SegmentInfo si = segmentInfos.info(i);
              if (si.getDocStoreOffset() != -1 &&
                  si.getDocStoreSegment().equals(docStoreSegment))
                si.setDocStoreIsCompoundFile(false);
            }
            deleter.deleteFile(compoundFileName);
            deletePartialSegmentsFile();
          }
        }
View Full Code Here

  /** Returns the number of documents currently in this index. */
  public synchronized int docCount() {
    ensureOpen();
    int count = docWriter.getNumDocsInRAM();
    for (int i = 0; i < segmentInfos.size(); i++) {
      SegmentInfo si = segmentInfos.info(i);
      count += si.docCount;
    }
    return count;
  }
View Full Code Here

        }

        SegmentInfos sis = new SegmentInfos(); // read infos from dir
        sis.read(dirs[i]);
        for (int j = 0; j < sis.size(); j++) {
          SegmentInfo info = sis.info(j);
          segmentInfos.addElement(info); // add each info
        }
      }

      maybeMerge();
View Full Code Here

TOP

Related Classes of uk.ac.ucl.panda.utility.structure.SegmentInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.