Package org.apache.drill.exec.record

Examples of org.apache.drill.exec.record.BatchSchema

32
33
34
35
36
37
38
39
40
41
42
  private RecordBatch batch;

  public VectorRecordMaterializer(FragmentContext context, RecordBatch batch) {
    this.queryId = context.getHandle().getQueryId();
    this.batch = batch;
    BatchSchema schema = batch.getSchema();
    assert schema != null : "Schema must be defined.";
   
//    for (MaterializedField f : batch.getSchema()) {
//      logger.debug("New Field: {}", f);
//    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  public void build(FragmentContext context) throws SchemaChangeException{
    container.clear();
    if(batches.keySet().size() > 1) throw new SchemaChangeException("Sort currently only supports a single schema.");
    if(batches.size() > Character.MAX_VALUE) throw new SchemaChangeException("Sort cannot work on more than %d batches at a time.", (int) Character.MAX_VALUE);
    sv4 = new SelectionVector4(svAllocator.getAllocation(), recordCount, Character.MAX_VALUE);
    BatchSchema schema = batches.keySet().iterator().next();
    List<RecordBatchData> data = batches.get(schema);
   
    // now we're going to generate the sv4 pointers
    switch(schema.getSelectionVectorMode()){
    case NONE: {
      int index = 0;
      int recordBatchId = 0;
      for(RecordBatchData d : data){
        for(int i =0; i < d.getRecordCount(); i++, index++){
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  public void build() throws SchemaChangeException {
    container.clear();
    if (queuedRightBatches.size() > Character.MAX_VALUE) throw new SchemaChangeException("Join cannot work on more than %d batches at a time.", (int) Character.MAX_VALUE);
    status.sv4 = new SelectionVector4(svAllocator.getAllocation(), recordCount, Character.MAX_VALUE);
    BatchSchema schema = queuedRightBatches.keySet().iterator().next();
    List<RecordBatchData> data = queuedRightBatches.get(schema);

    // now we're going to generate the sv4 pointers
    switch(schema.getSelectionVectorMode()){
      case NONE: {
        int index = 0;
        int recordBatchId = 0;
        for(RecordBatchData d : data){
          for(int i =0; i < d.getRecordCount(); i++, index++){
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    if(batches.size() > Character.MAX_VALUE) throw new SchemaChangeException("Sort cannot work on more than %d batches at a time.", (int) Character.MAX_VALUE);
    if(batches.keys().size() < 1){
      assert false : "Invalid to have an empty set of batches with no schemas.";
    }
    sv4 = new SelectionVector4(svAllocator.getAllocation(), recordCount, Character.MAX_VALUE);
    BatchSchema schema = batches.keySet().iterator().next();
    List<RecordBatchData> data = batches.get(schema);

    // now we're going to generate the sv4 pointers
    switch(schema.getSelectionVectorMode()){
    case NONE: {
      int index = 0;
      int recordBatchId = 0;
      for(RecordBatchData d : data){
        for(int i =0; i < d.getRecordCount(); i++, index++){
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    this.hasSv2 = hasSv2;
  }

  public void resetQueue(VectorContainer container, SelectionVector4 v4) throws SchemaChangeException {
    assert container.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.FOUR_BYTE;
    BatchSchema schema = container.getSchema();
    VectorContainer newContainer = new VectorContainer();
    for (MaterializedField field : schema) {
      int[] ids = container.getValueVectorId(field.getPath()).getFieldIds();
      newContainer.add(container.getValueAccessorById(field.getValueClass(), ids).getValueVectors());
    }
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
   * @throws Exception
   */
  protected void doQuery(FileInputStream input) throws Exception{
    int  batchNum = 0;
    int  emptyBatchNum = 0;
    BatchSchema prevSchema = null;
    List<Integer> schemaChangeIdx = Lists.newArrayList();

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
  private RecordBatch batch;

  public VectorRecordMaterializer(FragmentContext context, RecordBatch batch) {
    this.queryId = context.getHandle().getQueryId();
    this.batch = batch;
    BatchSchema schema = batch.getSchema();
    assert schema != null : "Schema must be defined.";
   
//    for (MaterializedField f : batch.getSchema()) {
//      logger.debug("New Field: {}", f);
//    }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
      throw new IllegalStateException("The incoming iterator returned a state of OK on the first batch. There should always be a new schema on the first batch. Incoming: " + incoming.getClass().getName());
    }
    if (first) first = !first;

    if(state == IterOutcome.OK || state == IterOutcome.OK_NEW_SCHEMA) {
      BatchSchema schema = incoming.getSchema();
      if(schema.getFieldCount() == 0){
        throw new IllegalStateException ("Incoming batch has an empty schema. This is not allowed.");
      }
      if(incoming.getRecordCount() > MAX_BATCH_SIZE){
        throw new IllegalStateException (String.format("Incoming batch of %s has size %d, which is beyond the limit of %d",  incoming.getClass().getName(), incoming.getRecordCount(), MAX_BATCH_SIZE));
      }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
  private boolean load(RecordBatchData batch) {
    VectorContainer newContainer = batch.getContainer();
    if (schema != null && newContainer.getSchema().equals(schema)) {
      container.zeroVectors();
      BatchSchema schema = container.getSchema();
      for (int i = 0; i < container.getNumberOfColumns(); i++) {
        MaterializedField field = schema.getColumn(i);
        MajorType type = field.getType();
        ValueVector vOut = container.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
                container.getValueVectorId(field.getPath()).getFieldIds()).getValueVector();
        ValueVector vIn = newContainer.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()),
                newContainer.getValueVectorId(field.getPath()).getFieldIds()).getValueVector();
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
533
534
535
  }

  private boolean isSameSchemaAmongBatches(RecordBatchLoader[] batchLoaders) {
    Preconditions.checkArgument(batchLoaders.length > 0, "0 batch is not allowed!");

    BatchSchema schema = batchLoaders[0].getSchema();

    for (int i = 1; i < batchLoaders.length; i++) {
      if (!schema.equals(batchLoaders[i].getSchema())) {
        logger.error("Schemas are different. Schema 1 : " + schema + ", Schema 2: " + batchLoaders[i].getSchema() );
        return false;
      }
    }
    return true;
View Full Code Here
TOP

Related Classes of org.apache.drill.exec.record.BatchSchema

  • org.apache.drill.exec.client.DumpCat
  • org.apache.drill.exec.physical.impl.join.MergeJoinBatchBuilder
  • org.apache.drill.exec.physical.impl.materialize.VectorRecordMaterializer
  • org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch
  • org.apache.drill.exec.physical.impl.producer.ProducerConsumerBatch
  • org.apache.drill.exec.physical.impl.sort.SortRecordBatchBuilder
  • org.apache.drill.exec.physical.impl.TopN.PriorityQueueTemplate
  • org.apache.drill.exec.physical.impl.validate.IteratorValidatorBatchIterator
  • org.apache.drill.exec.physical.impl.writer.TestParquetWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.