Package org.apache.drill.exec.record.selection

Examples of org.apache.drill.exec.record.selection.SelectionVector2

114
115
116
117
118
119
120
121
122
123
124
125
126
    }


    if (rbd.getRecordCount() == 0 && batches.size() > 0) {
      rbd.getContainer().zeroVectors();
      SelectionVector2 sv2 = rbd.getSv2();
      if (sv2 != null) {
        sv2.clear();
      }
      return true;
    }
    runningBytes += batchBytes;
    batches.put(rbd.getContainer().getSchema(), rbd);
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    }

    doSetup(context, hyperBatch, null); // may not need to do this every time

    int count = 0;
    SelectionVector2 sv2 = null;
    if (hasSv2) {
      sv2 = batch.getSv2();
    }
    for (; queueSize < limit && count < batch.getRecordCount();  count++) {
      heapSv4.set(queueSize, batchCount, hasSv2 ? sv2.getIndex(count) : count);
      queueSize++;
      siftUp();
    }
    for (; count < batch.getRecordCount(); count++) {
      heapSv4.set(limit, batchCount, hasSv2 ? sv2.getIndex(count) : count);
      if (compare(limit, 0) < 0) {
        swap(limit, 0);
        siftDown();
      }
    }
    batchCount++;
    if (hasSv2) {
      sv2.clear();
    }
    logger.debug("Took {} us to add {} records", watch.elapsed(TimeUnit.MICROSECONDS), count);
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    }

    switch (incoming.getSchema().getSelectionVectorMode()) {
      case NONE:
        if (sv2 == null) {
          sv2 = new SelectionVector2(oContext.getAllocator());
        }
        this.filter = generateSV2Filterer();
        break;
      case TWO_BYTE:
        sv2 = new SelectionVector2(oContext.getAllocator());
        this.filter = generateSV2Filterer();
        break;
      case FOUR_BYTE:
        /*
         * Filter does not support SV4 handling. There are couple of minor issues in the
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
    UserBitShared.RecordBatchDef batchDef = UserBitShared.RecordBatchDef.parseDelimitedFrom(input);
    recordCount = batchDef.getRecordCount();
    if (batchDef.hasCarriesTwoByteSelectionVector() && batchDef.getCarriesTwoByteSelectionVector()) {

      if (sv2 == null) {
        sv2 = new SelectionVector2(allocator);
      }
      sv2.allocateNew(recordCount * SelectionVector2.RECORD_SIZE);
      sv2.getBuffer().setBytes(0, input, recordCount * SelectionVector2.RECORD_SIZE);
      svMode = BatchSchema.SelectionVectorMode.TWO_BYTE;
    }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    UserBitShared.RecordBatchDef batchDef = UserBitShared.RecordBatchDef.parseDelimitedFrom(input);
    recordCount = batchDef.getRecordCount();
    if (batchDef.hasCarriesTwoByteSelectionVector() && batchDef.getCarriesTwoByteSelectionVector()) {

      if (sv2 == null) {
        sv2 = new SelectionVector2(allocator);
      }
      sv2.allocateNew(recordCount * SelectionVector2.RECORD_SIZE);
      sv2.getBuffer().setBytes(0, input, recordCount * SelectionVector2.RECORD_SIZE);
      svMode = BatchSchema.SelectionVectorMode.TWO_BYTE;
    }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
      sv2.clear();
    }

    switch(incoming.getSchema().getSelectionVectorMode()){
      case NONE:
        sv2 = new SelectionVector2(oContext.getAllocator());
        this.filter = generateSV2Filterer();
        break;
      case TWO_BYTE:
        sv2 = new SelectionVector2(oContext.getAllocator());
        this.filter = generateSV2Filterer();
        break;
      case FOUR_BYTE:
        /*
         * Filter does not support SV4 handling. There are couple of minor issues in the
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
    }


    if (rbd.getRecordCount() == 0 && batches.size() > 0) {
      rbd.getContainer().zeroVectors();
      SelectionVector2 sv2 = rbd.getSv2();
      if (sv2 != null) {
        sv2.clear();
      }
      return true;
    }
    runningBytes += batchBytes;
    batches.put(rbd.getContainer().getSchema(), rbd);
View Full Code Here
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
            }
            break;
          }
          if (first) first = false;
          totalSizeInMemory += getBufferSize(incoming);
          SelectionVector2 sv2;
          if (incoming.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.TWO_BYTE) {
            sv2 = incoming.getSelectionVector2();
            if (sv2.getBuffer(false).isRootBuffer()) {
              oContext.getAllocator().takeOwnership(sv2.getBuffer(false));
            }
          } else {
            try {
              sv2 = newSV2();
            } catch (OutOfMemoryException e) {
              throw new RuntimeException(e);
            }
          }
          int count = sv2.getCount();
          totalcount += count;
//          if (count == 0) {
//            break outer;
//          }
          sorter.setup(context, sv2, incoming);
View Full Code Here
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
    }
    return size;
  }

  private SelectionVector2 newSV2() throws OutOfMemoryException {
    SelectionVector2 sv2 = new SelectionVector2(oContext.getAllocator());
    if (!sv2.allocateNew(incoming.getRecordCount())) {
      try {
        mergeAndSpill();
      } catch (SchemaChangeException e) {
        throw new RuntimeException();
      }
      batchesSinceLastSpill = 0;
      int waitTime = 1;
      while (true) {
        try {
          Thread.sleep(waitTime * 1000);
        } catch (InterruptedException e) {
          throw new OutOfMemoryException(e);
        }
        waitTime *= 2;
        if (sv2.allocateNew(incoming.getRecordCount())) {
          break;
        }
        if (waitTime >= 32) {
          throw new OutOfMemoryException("Unable to allocate sv2 buffer after repeated attempts");
        }
      }
    }
    for (int i = 0; i < incoming.getRecordCount(); i++) {
      sv2.setIndex(i, (char) i);
    }
    sv2.setRecordCount(incoming.getRecordCount());
    return sv2;
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
  private boolean done = false;
  List<TransferPair> transfers = Lists.newArrayList();

  public LimitRecordBatch(Limit popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException {
    super(popConfig, context, incoming);
    outgoingSv = new SelectionVector2(oContext.getAllocator());
    recordsToSkip = popConfig.getFirst();
    noEndLimit = popConfig.getLast() == null;
    if(!noEndLimit) {
      recordsLeft = popConfig.getLast() - recordsToSkip;
    }
View Full Code Here
TOP

Related Classes of org.apache.drill.exec.record.selection.SelectionVector2

  • io.netty.buffer.ByteBuf
  • io.netty.buffer.DrillBuf
  • org.apache.drill.exec.cache.VectorAccessibleSerializable
  • org.apache.drill.exec.physical.impl.filter.FilterRecordBatch
  • org.apache.drill.exec.physical.impl.limit.LimitRecordBatch
  • org.apache.drill.exec.physical.impl.sort.SortRecordBatchBuilder
  • org.apache.drill.exec.physical.impl.TopN.PriorityQueueTemplate
  • org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.