Examples of VectorContainer

org.apache.drill.exec.record.VectorContainer

Examples of org.apache.drill.exec.record.VectorContainer

        vectors[i++] = group.getValueAccessorById(
            field.getValueClass(),
            group.getValueVectorId(field.getPath()).getFieldIds()
                ).getValueVector();
        if (group.hasSecond()) {
          VectorContainer c = group.getSecondContainer();
          vectors[i++] = c.getValueAccessorById(
              field.getValueClass(),
              c.getValueVectorId(field.getPath()).getFieldIds()
                  ).getValueVector();
        } else {
          vectors[i] = vectors[i - 1].getTransferPair().getTo(); //this vector should never be used. Just want to avoid having null elements in the hyper vector
          i++;
        }

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer


  public List<VectorContainer> getHeldRecordBatches() {
    ArrayList<VectorContainer> containerList = Lists.newArrayList();
    for (BatchSchema bs : batches.keySet()) {
      for (RecordBatchData bd : batches.get(bs)) {
        VectorContainer c = bd.getContainer();
        c.setRecordCount(bd.getRecordCount());
        containerList.add(c);
      }
    }
    batches.clear();
    return containerList;

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    vas.readFromStream(inputStream);
    VectorContainer c = (VectorContainer) vas.get();
//    logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    return c;
  }

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

      pair.transfer();
    }
    firstContainer.setRecordCount(secondContainer.getRecordCount());


    if (spilledBatches > 0) {
      VectorContainer c = null;
      try {
        c = getBatch();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      secondContainer.zeroVectors();
      wrapperIterator = c.iterator();
      for (VectorWrapper w : secondContainer) {
        TransferPair pair = wrapperIterator.next().getValueVector().makeTransferPair(w.getValueVector());
        pair.transfer();
      }
      secondContainer.setRecordCount(c.getRecordCount());
      c.zeroVectors();
    } else {
      secondContainer.zeroVectors();
      hasSecond = false;
    }
    batchPointer = 0;

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

  }


  public void resetQueue(VectorContainer container, SelectionVector4 v4) throws SchemaChangeException {
    assert container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.FOUR_BYTE;
    BatchSchema schema = container.getSchema();
    VectorContainer newContainer = new VectorContainer();
    for (MaterializedField field : schema) {
      int[] ids = container.getValueVectorId(field.getPath()).getFieldIds();
      newContainer.add(container.getValueAccessorById(field.getValueClass(), ids).getValueVectors());
    }
    newContainer.buildSchema(BatchSchema.SelectionVectorMode.FOUR_BYTE);
    this.hyperBatch = new ExpandableHyperContainer(newContainer);
    this.batchCount = hyperBatch.iterator().next().getValueVectors().length;
    BufferAllocator.PreAllocator preAlloc = allocator.getNewPreAllocator();
    preAlloc.preAllocate(4 * (limit + 1));
    this.heapSv4 = new SelectionVector4(preAlloc.getAllocation(), limit, Character.MAX_VALUE);

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    intVector.getMutator().setSafe(2, 2); binVector.getMutator().setSafe(2, "TWO".getBytes());
    intVector.getMutator().setSafe(3, 3); binVector.getMutator().setSafe(3, "THREE".getBytes());
    intVector.getMutator().setValueCount(4);
    binVector.getMutator().setValueCount(4);


    VectorContainer container = new VectorContainer();
    container.addCollection(vectorList);
    container.setRecordCount(4);
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(container.getRecordCount(), container, false);
    VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, context.getAllocator());


    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
    FileSystem fs = FileSystem.get(conf);

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();


    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
       VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();


       aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));


       if (vectorContainer.getRecordCount() == 0) {
         emptyBatchNum ++;
       }


       if (prevSchema != null && !vectorContainer.getSchema().equals(prevSchema))
         schemaChangeIdx.add(batchNum);


       prevSchema = vectorContainer.getSchema();
       batchNum ++;


       vectorContainer.zeroVectors();
    }


     /* output the summary stat */
     System.out.println(String.format("Total # of batches: %d", batchNum));
     //output: rows, selectedRows, avg rec size, total data size.

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);


      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }


    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }


    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    }
  }




  private void showSingleBatch (VectorAccessibleSerializable vcSerializable, boolean showHeader) {
    VectorContainer vectorContainer = (VectorContainer)vcSerializable.get();


    /* show the header of the batch */
    if (showHeader) {
      System.out.println(getBatchMetaInfo(vcSerializable).toString());

View Full Code Here

Examples of org.apache.drill.exec.record.VectorContainer

    intVector.getMutator().setSafe(3, 3);
    binVector.getMutator().setSafe(3, "THREE".getBytes());
    intVector.getMutator().setValueCount(4);
    binVector.getMutator().setValueCount(4);


    VectorContainer container = new VectorContainer();
    container.addCollection(vectorList);
    container.setRecordCount(4);
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(container.getRecordCount(), container, false);
    CachedVectorContainer wrap = new CachedVectorContainer(batch, ALLOCATOR);


    DistributedMultiMap<String, CachedVectorContainer> mmap = ICACHE.getMultiMap(OrderedPartitionRecordBatch.MULTI_CACHE_CONFIG);
    mmap.put("vectors", wrap).get();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.