Examples of org.apache.lucene.store.ByteArrayDataOutput

org.apache.lucene.store.ByteArrayDataOutput
@lucene.experimental


    boolean success = false;
    count = 0;
    byte buffer[] = new byte[8];
    try {
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
      BytesRef surfaceForm;


      while ((surfaceForm = iterator.next()) != null) {
        Set<IntsRef> paths = toFiniteStrings(surfaceForm, ts2a);
        
        maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, paths.size());


        for (IntsRef path : paths) {


          Util.toBytesRef(path, scratch);
          
          // length of the analyzed text (FST input)
          if (scratch.length > Short.MAX_VALUE-2) {
            throw new IllegalArgumentException("cannot handle analyzed forms > " + (Short.MAX_VALUE-2) + " in length (got " + scratch.length + ")");
          }
          short analyzedLength = (short) scratch.length;


          // compute the required length:
          // analyzed sequence + weight (4) + surface + analyzedLength (short)
          int requiredLength = analyzedLength + 4 + surfaceForm.length + 2;


          BytesRef payload;


          if (hasPayloads) {
            if (surfaceForm.length > (Short.MAX_VALUE-2)) {
              throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")");
            }
            payload = iterator.payload();
            // payload + surfaceLength (short)
            requiredLength += payload.length + 2;
          } else {
            payload = null;
          }
          
          buffer = ArrayUtil.grow(buffer, requiredLength);
          
          output.reset(buffer);


          output.writeShort(analyzedLength);


          output.writeBytes(scratch.bytes, scratch.offset, scratch.length);


          output.writeInt(encodeWeight(iterator.weight()));


          if (hasPayloads) {
            for(int i=0;i<surfaceForm.length;i++) {
              if (surfaceForm.bytes[i] == PAYLOAD_SEP) {
                throw new IllegalArgumentException("surface form cannot contain unit separator character U+001F; this character is reserved");
              }
            }
            output.writeShort((short) surfaceForm.length);
            output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
            output.writeBytes(payload.bytes, payload.offset, payload.length);
          } else {
            output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
          }


          assert output.getPosition() == requiredLength: output.getPosition() + " vs " + requiredLength;


          writer.write(buffer, 0, output.getPosition());
        }
        count++;
      }
      writer.close();

View Full Code Here

    final OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
    boolean success = false;
    try {
      BytesRef spare;
      byte[] buffer = new byte[0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);


      while ((spare = source.next()) != null) {
        encode(writer, output, buffer, spare, source.payload(), source.contexts(), source.weight());
      }
      writer.close();

View Full Code Here

    final ByteSequencesWriter writer = new ByteSequencesWriter(tempInput);
    boolean success = false;
    try {
      BytesRef spare;
      byte[] buffer = new byte[0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);


      while ((spare = source.next()) != null) {
        encode(writer, output, buffer, spare, source.weight());
      }
      writer.close();

View Full Code Here

    // Push floats up front before sequences to sort them. For now, assume they are non-negative.
    // If negative floats are allowed some trickery needs to be done to find their byte order.
    boolean success = false;
    try {
      byte [] buffer = new byte [0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
      BytesRef spare;
      while ((spare = tfit.next()) != null) {
        if (spare.length + 4 >= buffer.length) {
          buffer = ArrayUtil.grow(buffer, spare.length + 4);
        }


        output.reset(buffer);
        output.writeInt(encodeWeight(tfit.weight()));
        output.writeBytes(spare.bytes, spare.offset, spare.length);
        writer.write(buffer, 0, output.getPosition());
      }
      writer.close();


      // We don't know the distribution of scores and we need to bucket them, so we'll sort
      // and divide into equal buckets.

View Full Code Here

    final Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    boolean success = false;
    try {
      BytesRef spare;
      byte[] buffer = new byte[0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);


      while ((spare = source.next()) != null) {
        encode(writer, output, buffer, spare, source.weight());
      }
      writer.close();

View Full Code Here

    
    Comparator<BytesRef> comparator = BytesRef.getUTF8SortedAsUnicodeComparator();
    BytesRefHash sorted = new BytesRefHash();
    TermFreq[] unsorted = new TermFreq[num];
    byte[] buffer = new byte[0];
    ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);


    for (int i = 0; i < num; i++) {
      BytesRef spare;
      long weight;
      do {
        spare = new BytesRef(_TestUtil.randomUnicodeString(random));
        if (spare.length + 8 >= buffer.length) {
          buffer = ArrayUtil.grow(buffer, spare.length + 8);
        }
        output.reset(buffer);
        output.writeBytes(spare.bytes, spare.offset, spare.length);
        weight = random.nextLong();
        output.writeLong(weight);
        
      } while (sorted.add(new BytesRef(buffer, 0, output.getPosition())) < 0);
      unsorted[i] = new TermFreq(spare, weight);
    }
    
    // test the sorted iterator wrapper
    TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), comparator, true);

View Full Code Here

      // TODO: are we using the best sharing options?
      org.apache.lucene.util.fst.Builder<BytesRef> builder = 
        new org.apache.lucene.util.fst.Builder<BytesRef>(FST.INPUT_TYPE.BYTE4, outputs);
      
      BytesRef scratch = new BytesRef(64);
      ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput();


      final Set<Integer> dedupSet;


      if (dedup) {
        dedupSet = new HashSet<Integer>();
      } else {
        dedupSet = null;
      }


      final byte[] spare = new byte[5];
      
      Set<CharsRef> keys = workingSet.keySet();
      CharsRef sortedKeys[] = keys.toArray(new CharsRef[keys.size()]);
      Arrays.sort(sortedKeys, CharsRef.getUTF16SortedAsUTF8Comparator());
      
      final IntsRef scratchIntsRef = new IntsRef();


      //System.out.println("fmap.build");
      for (int keyIdx = 0; keyIdx < sortedKeys.length; keyIdx++) {
        CharsRef input = sortedKeys[keyIdx];
        MapEntry output = workingSet.get(input);


        int numEntries = output.ords.size();
        // output size, assume the worst case
        int estimatedSize = 5 + numEntries * 5; // numEntries + one ord for each entry
        
        scratch.grow(estimatedSize);
        scratchOutput.reset(scratch.bytes, scratch.offset, scratch.bytes.length);
        assert scratch.offset == 0;


        // now write our output data:
        int count = 0;
        for (int i = 0; i < numEntries; i++) {
          if (dedupSet != null) {
            // box once
            final Integer ent = output.ords.get(i);
            if (dedupSet.contains(ent)) {
              continue;
            }
            dedupSet.add(ent);
          }
          scratchOutput.writeVInt(output.ords.get(i));   
          count++;
        }


        final int pos = scratchOutput.getPosition();
        scratchOutput.writeVInt(count << 1 | (output.includeOrig ? 0 : 1));
        final int pos2 = scratchOutput.getPosition();
        final int vIntLen = pos2-pos;


        // Move the count + includeOrig to the front of the byte[]:
        System.arraycopy(scratch.bytes, pos, spare, 0, vIntLen);
        System.arraycopy(scratch.bytes, 0, scratch.bytes, vIntLen, pos);
        System.arraycopy(spare, 0, scratch.bytes, 0, vIntLen);


        if (dedupSet != null) {
          dedupSet.clear();
        }
        
        scratch.length = scratchOutput.getPosition() - scratch.offset;
        //System.out.println("  add input=" + input + " output=" + scratch + " offset=" + scratch.offset + " length=" + scratch.length + " count=" + count);
        builder.add(Util.toUTF32(input, scratchIntsRef), BytesRef.deepCopyOf(scratch));
      }
      
      FST<BytesRef> fst = builder.finish();

View Full Code Here

    final Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
    boolean success = false;
    try {
      BytesRef spare;
      byte[] buffer = new byte[0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);


      while ((spare = source.next()) != null) {
        encode(writer, output, buffer, spare, source.weight());
      }
      writer.close();

View Full Code Here

    // Push floats up front before sequences to sort them. For now, assume they are non-negative.
    // If negative floats are allowed some trickery needs to be done to find their byte order.
    boolean success = false;
    try {
      byte [] buffer = new byte [0];
      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
      BytesRef spare;
      while ((spare = tfit.next()) != null) {
        if (spare.length + 4 >= buffer.length) {
          buffer = ArrayUtil.grow(buffer, spare.length + 4);
        }


        output.reset(buffer);
        output.writeInt(encodeWeight(tfit.weight()));
        output.writeBytes(spare.bytes, spare.offset, spare.length);
        writer.write(buffer, 0, output.getPosition());
      }
      writer.close();


      // We don't know the distribution of scores and we need to bucket them, so we'll sort
      // and divide into equal buckets.

View Full Code Here

        public BytesRef binaryValue() {
            try {
                CollectionUtils.sortAndDedup(bytesList);
                int size = bytesList.size();
                final byte[] bytes = new byte[totalSize + (size + 1) * 5];
                ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
                out.writeVInt(size);  // write total number of values
                for (int i = 0; i < size; i ++) {
                    final byte[] value = bytesList.get(i);
                    int valueLength = value.length;
                    out.writeVInt(valueLength);
                    out.writeBytes(value, 0, valueLength);
                }
                return new BytesRef(bytes, 0, out.getPosition());
            } catch (IOException e) {
                throw new ElasticsearchException("Failed to get binary value", e);
            }


        }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.lucene.store.ByteArrayDataOutput

org.apache.lucene.analysis.hunspell.Dictionary

org.apache.lucene.analysis.synonym.SynonymMap$Builder

org.apache.lucene.codecs.compressing.AbstractTestCompressionMode

org.apache.lucene.index.Test2BBinaryDocValues

org.apache.lucene.index.TestIndexInput

org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester

org.apache.lucene.search.suggest.fst.FSTCompletionLookup

org.apache.lucene.search.suggest.SortedInputIterator

org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper

org.apache.lucene.search.suggest.TestTermFreqIterator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.