Package org.apache.lucene.store

Examples of org.apache.lucene.store.BufferedChecksumIndexInput


   * If you just need to extract the checksum value, call {@link #retrieveChecksum}.
   */
  public static long checksumEntireFile(IndexInput input) throws IOException {
    IndexInput clone = input.clone();
    clone.seek(0);
    ChecksumIndexInput in = new BufferedChecksumIndexInput(clone);
    assert in.getFilePointer() == 0;
    in.seek(in.length() - footerLength());
    return checkFooter(in);
  }
View Full Code Here


      numStoredFields = new int[1];
      lengths = new int[1];

      IndexInput in = CompressingStoredFieldsReader.this.fieldsStream;
      in.seek(0);
      fieldsStream = new BufferedChecksumIndexInput(in);
      fieldsStream.seek(indexReader.getStartPointer(startDocId));
    }
View Full Code Here

  @Override
  public void checkIntegrity() throws IOException {
    BytesRef scratch = new BytesRef();
    IndexInput clone = data.clone();
    clone.seek(0);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while(true) {
      SimpleTextUtil.readLine(input, scratch);
      if (scratch.equals(END)) {
        SimpleTextUtil.checkFooter(input);
        break;
View Full Code Here

      }
    }
  }
 
  private TreeMap<String,Long> readFields(IndexInput in) throws IOException {
    ChecksumIndexInput input = new BufferedChecksumIndexInput(in);
    BytesRef scratch = new BytesRef(10);
    TreeMap<String,Long> fields = new TreeMap<>();
   
    while (true) {
      SimpleTextUtil.readLine(input, scratch);
      if (scratch.equals(END)) {
        SimpleTextUtil.checkFooter(input);
        return fields;
      } else if (StringHelper.startsWith(scratch, FIELD)) {
        String fieldName = new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, StandardCharsets.UTF_8);
        fields.put(fieldName, input.getFilePointer());
      }
    }
  }
View Full Code Here

 
  // we don't actually write a .tvx-like index, instead we read the
  // vectors file in entirety up-front and save the offsets
  // so we can seek to the data later.
  private void readIndex(int maxDoc) throws IOException {
    ChecksumIndexInput input = new BufferedChecksumIndexInput(in);
    offsets = new long[maxDoc];
    int upto = 0;
    while (!scratch.equals(END)) {
      SimpleTextUtil.readLine(input, scratch);
      if (StringHelper.startsWith(scratch, DOC)) {
        offsets[upto] = input.getFilePointer();
        upto++;
      }
    }
    SimpleTextUtil.checkFooter(input);
    assert upto == offsets.length;
View Full Code Here

 
  // we don't actually write a .fdx-like index, instead we read the
  // stored fields file in entirety up-front and save the offsets
  // so we can seek to the documents later.
  private void readIndex(int size) throws IOException {
    ChecksumIndexInput input = new BufferedChecksumIndexInput(in);
    offsets = new long[size];
    int upto = 0;
    while (!scratch.equals(END)) {
      SimpleTextUtil.readLine(input, scratch);
      if (StringHelper.startsWith(scratch, DOC)) {
        offsets[upto] = input.getFilePointer();
        upto++;
      }
    }
    SimpleTextUtil.checkFooter(input);
    assert upto == offsets.length;
View Full Code Here

   * If you just need to extract the checksum value, call {@link #retrieveChecksum}.
   */
  public static long checksumEntireFile(IndexInput input) throws IOException {
    IndexInput clone = input.clone();
    clone.seek(0);
    ChecksumIndexInput in = new BufferedChecksumIndexInput(clone);
    assert in.getFilePointer() == 0;
    in.seek(in.length() - footerLength());
    return checkFooter(in);
  }
View Full Code Here

      numStoredFields = new int[1];
      lengths = new int[1];

      IndexInput in = CompressingStoredFieldsReader.this.fieldsStream;
      in.seek(0);
      fieldsStream = new BufferedChecksumIndexInput(in);
      fieldsStream.seek(indexReader.getStartPointer(startDocId));
    }
View Full Code Here

        }
      } else {
        final CompressingStoredFieldsIndexReader index = matchingVectorsReader.getIndex();
        final IndexInput vectorsStreamOrig = matchingVectorsReader.getVectorsStream();
        vectorsStreamOrig.seek(0);
        final ChecksumIndexInput vectorsStream = new BufferedChecksumIndexInput(vectorsStreamOrig.clone());
       
        for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; ) {
          // We make sure to move the checksum input in any case, otherwise the final
          // integrity check might need to read the whole file a second time
          final long startPointer = index.getStartPointer(i);
          if (startPointer > vectorsStream.getFilePointer()) {
            vectorsStream.seek(startPointer);
          }
          if (pendingDocs.isEmpty()
              && (i == 0 || index.getStartPointer(i - 1) < startPointer)) { // start of a chunk
            final int docBase = vectorsStream.readVInt();
            final int chunkDocs = vectorsStream.readVInt();
            assert docBase + chunkDocs <= matchingSegmentReader.maxDoc();
            if (docBase + chunkDocs < matchingSegmentReader.maxDoc()
                && nextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs) {
              final long chunkEnd = index.getStartPointer(docBase + chunkDocs);
              final long chunkLength = chunkEnd - vectorsStream.getFilePointer();
              indexWriter.writeIndex(chunkDocs, this.vectorsStream.getFilePointer());
              this.vectorsStream.writeVInt(docCount);
              this.vectorsStream.writeVInt(chunkDocs);
              this.vectorsStream.copyBytes(vectorsStream, chunkLength);
              docCount += chunkDocs;
              this.numDocs += chunkDocs;
              mergeState.checkAbort.work(300 * chunkDocs);
              i = nextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc);
            } else {
              for (; i < docBase + chunkDocs; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
                final Fields vectors = reader.getTermVectors(i);
                addAllDocVectors(vectors, mergeState);
                ++docCount;
                mergeState.checkAbort.work(300);
              }
            }
          } else {
            final Fields vectors = reader.getTermVectors(i);
            addAllDocVectors(vectors, mergeState);
            ++docCount;
            mergeState.checkAbort.work(300);
            i = nextLiveDoc(i + 1, liveDocs, maxDoc);
          }
        }
       
        vectorsStream.seek(vectorsStream.length() - CodecUtil.footerLength());
        CodecUtil.checkFooter(vectorsStream);
      }
    }
    finish(mergeState.fieldInfos, docCount);
    return docCount;
View Full Code Here

  @Override
  public void checkIntegrity() throws IOException {
    BytesRef scratch = new BytesRef();
    IndexInput clone = data.clone();
    clone.seek(0);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while(true) {
      SimpleTextUtil.readLine(input, scratch);
      if (scratch.equals(END)) {
        SimpleTextUtil.checkFooter(input);
        break;
View Full Code Here

TOP

Related Classes of org.apache.lucene.store.BufferedChecksumIndexInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.