Package org.apache.drill.exec.record

Examples of org.apache.drill.exec.record.VectorContainer

420
421
422
423
424
425
426
427
428
429
430
431
432
433
        vectors[i++] = group.getValueAccessorById(
            field.getValueClass(),
            group.getValueVectorId(field.getPath()).getFieldIds()
                ).getValueVector();
        if (group.hasSecond()) {
          VectorContainer c = group.getSecondContainer();
          vectors[i++] = c.getValueAccessorById(
              field.getValueClass(),
              c.getValueVectorId(field.getPath()).getFieldIds()
                  ).getValueVector();
        } else {
          vectors[i] = vectors[i - 1].getTransferPair().getTo(); //this vector should never be used. Just want to avoid having null elements in the hyper vector
          i++;
        }
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
  public List<VectorContainer> getHeldRecordBatches() {
    ArrayList<VectorContainer> containerList = Lists.newArrayList();
    for (BatchSchema bs : batches.keySet()) {
      for (RecordBatchData bd : batches.get(bs)) {
        VectorContainer c = bd.getContainer();
        c.setRecordCount(bd.getRecordCount());
        containerList.add(c);
      }
    }
    batches.clear();
    return containerList;
View Full Code Here
105
106
107
108
109
110
111
112
113
114
    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    vas.readFromStream(inputStream);
    VectorContainer c = (VectorContainer) vas.get();
//    logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    return c;
  }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
      pair.transfer();
    }
    firstContainer.setRecordCount(secondContainer.getRecordCount());

    if (spilledBatches > 0) {
      VectorContainer c = null;
      try {
        c = getBatch();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      secondContainer.zeroVectors();
      wrapperIterator = c.iterator();
      for (VectorWrapper w : secondContainer) {
        TransferPair pair = wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
        pair.transfer();
      }
      secondContainer.setRecordCount(c.getRecordCount());
      c.zeroVectors();
    } else {
      secondContainer.zeroVectors();
      hasSecond = false;
    }
    batchPointer = 0;
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  }

  public void resetQueue(VectorContainer container, SelectionVector4 v4) throws SchemaChangeException {
    assert container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.FOUR_BYTE;
    BatchSchema schema = container.getSchema();
    VectorContainer newContainer = new VectorContainer();
    for (MaterializedField field : schema) {
      int[] ids = container.getValueVectorId(field.getPath()).getFieldIds();
      newContainer.add(container.getValueAccessorById(field.getValueClass(), ids).getValueVectors());
    }
    newContainer.buildSchema(BatchSchema.SelectionVectorMode.FOUR_BYTE);
    this.hyperBatch = new ExpandableHyperContainer(newContainer);
    this.batchCount = hyperBatch.iterator().next().getValueVectors().length;
    BufferAllocator.PreAllocator preAlloc = allocator.getNewPreAllocator();
    preAlloc.preAllocate(4 * (limit + 1));
    this.heapSv4 = new SelectionVector4(preAlloc.getAllocation(), limit, Character.MAX_VALUE);
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    intVector.getMutator().setSafe(2, 2); binVector.getMutator().setSafe(2, "TWO".getBytes());
    intVector.getMutator().setSafe(3, 3); binVector.getMutator().setSafe(3, "THREE".getBytes());
    intVector.getMutator().setValueCount(4);
    binVector.getMutator().setValueCount(4);

    VectorContainer container = new VectorContainer();
    container.addCollection(vectorList);
    container.setRecordCount(4);
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(container.getRecordCount(), container, false);
    VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, context.getAllocator());

    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
    FileSystem fs = FileSystem.get(conf);
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
       VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();

       aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));

       if (vectorContainer.getRecordCount() == 0) {
         emptyBatchNum ++;
       }

       if (prevSchema != null && !vectorContainer.getSchema().equals(prevSchema))
         schemaChangeIdx.add(batchNum);

       prevSchema = vectorContainer.getSchema();
       batchNum ++;

       vectorContainer.zeroVectors();
    }

     /* output the summary stat */
     System.out.println(String.format("Total # of batches: %d", batchNum));
     //output: rows, selectedRows, avg rec size, total data size.
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);

      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }

    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }

    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
    }
  }


  private void showSingleBatch (VectorAccessibleSerializable vcSerializable, boolean showHeader) {
    VectorContainer vectorContainer = (VectorContainer)vcSerializable.get();

    /* show the header of the batch */
    if (showHeader) {
      System.out.println(getBatchMetaInfo(vcSerializable).toString());

View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    intVector.getMutator().setSafe(3, 3);
    binVector.getMutator().setSafe(3, "THREE".getBytes());
    intVector.getMutator().setValueCount(4);
    binVector.getMutator().setValueCount(4);

    VectorContainer container = new VectorContainer();
    container.addCollection(vectorList);
    container.setRecordCount(4);
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(container.getRecordCount(), container, false);
    CachedVectorContainer wrap = new CachedVectorContainer(batch, ALLOCATOR);

    DistributedMultiMap<String, CachedVectorContainer> mmap = ICACHE.getMultiMap(OrderedPartitionRecordBatch.MULTI_CACHE_CONFIG);
    mmap.put("vectors", wrap).get();
View Full Code Here
TOP

Related Classes of org.apache.drill.exec.record.VectorContainer

  • org.apache.drill.exec.cache.TestCacheSerialization
  • org.apache.drill.exec.cache.TestWriteToDisk
  • org.apache.drill.exec.cache.VectorAccessibleSerializable
  • org.apache.drill.exec.client.DumpCat
  • org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate$BatchHolder
  • org.apache.drill.exec.physical.impl.common.ChainedHashTable
  • org.apache.drill.exec.physical.impl.common.HashTableTemplate$BatchHolder
  • org.apache.drill.exec.physical.impl.join.HashJoinBatch
  • org.apache.drill.exec.physical.impl.join.MergeJoinBatchBuilder
  • org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.