Package uk.ac.ucl.panda.utility.structure

Examples of uk.ac.ucl.panda.utility.structure.FieldInfo


        }

        // Discard pending norms:
        final int numField = fieldInfos.size();
        for (int i=0;i<numField;i++) {
          FieldInfo fi = fieldInfos.fieldInfo(i);
          if (fi.isIndexed() && !fi.getOmitNorms()) {
            BufferedNorms n = norms[i];
            if (n != null)
              try {
                n.reset();
              } catch (Throwable t) {
View Full Code Here


      normsOut.writeBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.length);

      final int numField = fieldInfos.size();

      for (int fieldIdx=0;fieldIdx<numField;fieldIdx++) {
        FieldInfo fi = fieldInfos.fieldInfo(fieldIdx);
        if (fi.isIndexed() && !fi.getOmitNorms()) {
          BufferedNorms n = norms[fieldIdx];
          final long v;
          if (n == null)
            v = 0;
          else {
View Full Code Here

      // vectors, etc.):

      for(int i=0;i<numDocFields;i++) {
        Fieldable field = (Fieldable) docFields.get(i);

        FieldInfo fi = fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(),
                                      field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
                                      field.getOmitNorms(), false);
        if (fi.isIndexed() && !fi.getOmitNorms()) {
          // Maybe grow our buffered norms
          if (norms.length <= fi.getNum()) {
            int newSize = (int) ((1+fi.getNum())*1.25);
            BufferedNorms[] newNorms = new BufferedNorms[newSize];
            System.arraycopy(norms, 0, newNorms, 0, norms.length);
            norms = newNorms;
          }
         
          if (norms[fi.getNum()] == null)
            norms[fi.getNum()] = new BufferedNorms();

          hasNorms = true;
        }

        // Make sure we have a FieldData allocated
        int hashPos = fi.getName().hashCode() & fieldDataHashMask;
        FieldData fp = fieldDataHash[hashPos];
        while(fp != null && !fp.fieldInfo.getName().equals(fi.getName()))
          fp = fp.next;

        if (fp == null) {

          fp = new FieldData(fi);
          fp.next = fieldDataHash[hashPos];
          fieldDataHash[hashPos] = fp;

          if (numAllFieldData == allFieldDataArray.length) {
            int newSize = (int) (allFieldDataArray.length*1.5);
            int newHashSize = fieldDataHash.length*2;

            FieldData newArray[] = new FieldData[newSize];
            FieldData newHashArray[] = new FieldData[newHashSize];
            System.arraycopy(allFieldDataArray, 0, newArray, 0, numAllFieldData);

            // Rehash
            fieldDataHashMask = newSize-1;
            for(int j=0;j<fieldDataHash.length;j++) {
              FieldData fp0 = fieldDataHash[j];
              while(fp0 != null) {
                hashPos = fp0.fieldInfo.getName().hashCode() & fieldDataHashMask;
                FieldData nextFP0 = fp0.next;
                fp0.next = newHashArray[hashPos];
                newHashArray[hashPos] = fp0;
                fp0 = nextFP0;
              }
            }

            allFieldDataArray = newArray;
            fieldDataHash = newHashArray;
          }
          allFieldDataArray[numAllFieldData++] = fp;
        } else {
          assert fp.fieldInfo == fi;
        }

        if (thisFieldGen != fp.lastGen) {

          // First time we're seeing this field for this doc
          fp.lastGen = thisFieldGen;
          fp.fieldCount = 0;
          fp.doVectors = fp.doVectorPositions = fp.doVectorOffsets = false;
          fp.doNorms = fi.isIndexed() && !fi.getOmitNorms();

          if (numFieldData == fieldDataArray.length) {
            int newSize = fieldDataArray.length*2;
            FieldData newArray[] = new FieldData[newSize];
            System.arraycopy(fieldDataArray, 0, newArray, 0, numFieldData);
View Full Code Here

    Document doc = new Document();
    int numFields = fieldsStream.readVInt();
    for (int i = 0; i < numFields; i++) {
      int fieldNumber = fieldsStream.readVInt();
      FieldInfo fi = fieldInfos.fieldInfo(fieldNumber);
      FieldSelectorResult acceptField = fieldSelector == null ? FieldSelectorResult.LOAD : fieldSelector.accept(fi.getName());
     
      byte bits = fieldsStream.readByte();
      assert bits <= FieldsWriter.FIELD_IS_COMPRESSED + FieldsWriter.FIELD_IS_TOKENIZED + FieldsWriter.FIELD_IS_BINARY;

      boolean compressed = (bits & FieldsWriter.FIELD_IS_COMPRESSED) != 0;
View Full Code Here

        files.add(segment + "." + ext);
    }

    // Fieldable norm files
    for (int i = 0; i < fieldInfos.size(); i++) {
      FieldInfo fi = fieldInfos.fieldInfo(i);
      if (fi.isIndexed() && !fi.getOmitNorms()) {
        files.add(segment + "." + IndexFileNames.NORMS_EXTENSION);
        break;
      }
    }
View Full Code Here

    for (int i = 0; i < readers.size(); i++) {
      IndexReader reader = (IndexReader) readers.elementAt(i);
      if (reader instanceof SegmentReader) {
        SegmentReader segmentReader = (SegmentReader) reader;
        for (int j = 0; j < segmentReader.getFieldInfos().size(); j++) {
          FieldInfo fi = segmentReader.getFieldInfos().fieldInfo(j);
          fieldInfos.add(fi.getName(), fi.isIndexed(), fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.getName()), fi.storePayloads);
        }
      } else {
        addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false);
        addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false);
        addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false);
View Full Code Here

  private void mergeNorms() throws IOException {
    byte[] normBuffer = null;
    IndexOutput output = null;
    try {
      for (int i = 0; i < fieldInfos.size(); i++) {
        FieldInfo fi = fieldInfos.fieldInfo(i);
        if (fi.isIndexed() && !fi.getOmitNorms()) {
          if (output == null) {
            output = directory.createOutput(segment + "." + IndexFileNames.NORMS_EXTENSION);
            output.writeBytes(NORMS_HEADER,NORMS_HEADER.length);
          }
          for (int j = 0; j < readers.size(); j++) {
            IndexReader reader = (IndexReader) readers.elementAt(j);
            int maxDoc = reader.maxDoc();
            if (normBuffer == null || normBuffer.length < maxDoc) {
              // the buffer is too small for the current segment
              normBuffer = new byte[maxDoc];
            }
            reader.norms(fi.getName(), normBuffer, 0);
            if (!reader.hasDeletions()) {
              //optimized case for segments without deleted docs
              output.writeBytes(normBuffer, maxDoc);
            } else {
              // this segment has deleted docs, so we have to
View Full Code Here

TOP

Related Classes of uk.ac.ucl.panda.utility.structure.FieldInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.