Package org.apache.drill.exec.vector

Examples of org.apache.drill.exec.vector.ValueVector


    // print headers.
    if (schemaChanged) {
    } // do not believe any change is needed for when the schema changes, with the current mock scan use case

    for (VectorWrapper vw : batchLoader) {
      ValueVector vv = vw.getValueVector();
      currentField = props.fields.get(vv.getField().getAsSchemaPath().getRootSegment().getPath());
      if (ParquetRecordReaderTest.VERBOSE_DEBUG){
        System.out.println("\n" + vv.getField().getAsSchemaPath().getRootSegment().getPath());
      }
      if ( ! valuesChecked.containsKey(vv.getField().getAsSchemaPath().getRootSegment().getPath())){
        valuesChecked.put(vv.getField().getAsSchemaPath().getRootSegment().getPath(), 0);
        columnValCounter = 0;
      } else {
        columnValCounter = valuesChecked.get(vv.getField().getAsSchemaPath().getRootSegment().getPath());
      }
      for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
        if (ParquetRecordReaderTest.VERBOSE_DEBUG){
          Object o = vv.getAccessor().getObject(j);
          if (o instanceof byte[]) {
            try {
              o = new String((byte[])o, "UTF-8");
            } catch (UnsupportedEncodingException e) {
              throw new RuntimeException(e);
            }
          }
          System.out.print(Strings.padStart(o + "", 20, ' ') + " ");
          System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
        }
        if (testValues){
          assertField(vv, j, currentField.type,
              currentField.values[columnValCounter % 3], currentField.name + "/");
        }
        columnValCounter++;
      }
      if (ParquetRecordReaderTest.VERBOSE_DEBUG){
        System.out.println("\n" + vv.getAccessor().getValueCount());
      }
      valuesChecked.remove(vv.getField().getAsSchemaPath().getRootSegment().getPath());
      if (valueCount == -1) {
        valueCount = columnValCounter;
      }
      else {
        assertEquals("Mismatched value count for vectors in the same batch.", valueCount, columnValCounter);
      }
      valuesChecked.put(vv.getField().getAsSchemaPath().getRootSegment().getPath(), columnValCounter);
    }

    if (ParquetRecordReaderTest.VERBOSE_DEBUG){
      for (int i = 0; i < batchLoader.getRecordCount(); i++) {
        recordCount++;
        if (i % 50 == 0){
          System.out.println();
          for (VectorWrapper vw : batchLoader) {
            ValueVector v = vw.getValueVector();
            System.out.print(Strings.padStart(v.getField().getAsSchemaPath().getRootSegment().getPath(), 20, ' ') + " ");

          }
          System.out.println();
          System.out.println();
        }

        for (VectorWrapper vw : batchLoader) {
          ValueVector v = vw.getValueVector();
          Object o = v.getAccessor().getObject(i);
          if (o instanceof byte[]) {
            try {
              // TODO - in the dictionary read error test there is some data that does not look correct
              // the output of our reader matches the values of the parquet-mr cat/head tools (no full comparison was made,
              // but from a quick check of a few values it looked consistent
View Full Code Here


    List<Object> res = new ArrayList<Object>();
    RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
    for(QueryResultBatch result : resultList) {
      if (result.getData() != null) {
        loader.load(result.getHeader().getDef(), result.getData());
        ValueVector v = loader.iterator().next().getValueVector();
        for (int j = 0; j < v.getAccessor().getValueCount(); j++) {
          if  (v instanceof VarCharVector) {
            res.add(new String(((VarCharVector) v).getAccessor().get(j)));
          } else {
            res.add(v.getAccessor().getObject(j));
          }
        }
        loader.clear();
        result.release();
      }
View Full Code Here

          isAnyWildcard = true;
          Integer value = result.prefixMap.get(result.prefix);
          if (value != null && value.intValue() == 1) {
            int k = 0;
            for(VectorWrapper<?> wrapper : incoming) {
              ValueVector vvIn = wrapper.getValueVector();
              SchemaPath originalPath = vvIn.getField().getPath();
              if (k > result.outputNames.size()-1) {
                assert false;
              }
              String name = result.outputNames.get(k++)// get the renamed column names
              if (name == EMPTY_STRING) continue;
              FieldReference ref = new FieldReference(name);
              TransferPair tp = wrapper.getValueVector().getTransferPair(ref);
              transfers.add(tp);
              container.add(tp.getTo());       
            }
          } else if (value != null && value.intValue() > 1) { // subsequent wildcards should do a copy of incoming valuevectors
            int k = 0;
            for(VectorWrapper<?> wrapper : incoming) {
              ValueVector vvIn = wrapper.getValueVector();
              SchemaPath originalPath = vvIn.getField().getPath();
              if (k > result.outputNames.size()-1) {
                assert false;
              }
              String name = result.outputNames.get(k++)// get the renamed column names
              if (name == EMPTY_STRING) continue;

              final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry() );
              if(collector.hasErrors()){
                throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
              }             

              MaterializedField outputField = MaterializedField.create(name, expr.getMajorType());
              ValueVector vv = TypeHelper.getNewVector(outputField, oContext.getAllocator());
              allocationVectors.add(vv);
              TypedFieldId fid = container.add(vv);
              ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
              HoldingContainer hc = cg.addExpr(write);

              cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
            }
          }
          continue;
        }
      }

      String outputName = getRef(namedExpression).getRootSegment().getPath();
      if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
        if (result.outputNames.get(0) == EMPTY_STRING) continue;
        outputName = result.outputNames.get(0);
      }
     
      final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
      final MaterializedField outputField = MaterializedField.create(outputName, expr.getMajorType());
      if(collector.hasErrors()){
        throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
      }

      // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
      if(expr instanceof ValueVectorReadExpression && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE
          && !((ValueVectorReadExpression) expr).hasReadPath()
          && !isAnyWildcard
          && !transferFieldIds.contains(((ValueVectorReadExpression) expr).getFieldId().getFieldIds()[0])
        ) {

        ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
        TypedFieldId id = vectorRead.getFieldId();
        ValueVector vvIn = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
        Preconditions.checkNotNull(incoming);

        TransferPair tp = vvIn.getTransferPair(getRef(namedExpression));
        transfers.add(tp);
        container.add(tp.getTo());
        transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        logger.debug("Added transfer for project expression.");
      } else if (expr instanceof DrillFuncHolderExpr &&
          ((DrillFuncHolderExpr) expr).isComplexWriterFuncHolder())  {
        // Need to process ComplexWriter function evaluation.
        // Lazy initialization of the list of complex writers, if not done yet.
        if (complexWriters == null)
          complexWriters = Lists.newArrayList();

        // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
        ((DrillComplexWriterFuncHolder) ((DrillFuncHolderExpr) expr).getHolder()).setReference(namedExpression.getRef());
        cg.addExpr(expr);
      } else{
        // need to do evaluation.
        ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
        allocationVectors.add(vector);
        TypedFieldId fid = container.add(vector);
        ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
        HoldingContainer hc = cg.addExpr(write);
View Full Code Here

      String[] components = ref.getPath().split(StarColumnHelper.PREFIX_DELIMITER, 2);
      assert(components.length == 2);
      String prefix = components[0];
      result.outputNames = Lists.newArrayList();
      for(VectorWrapper<?> wrapper : incoming) {
        ValueVector vvIn = wrapper.getValueVector();
        String name = vvIn.getField().getPath().getRootSegment().getPath();

        // add the prefix to the incoming column name
        String newName = prefix + StarColumnHelper.PREFIX_DELIMITER + name;
        addToResultMaps(newName, result, false);
      }
    }
    // input and output are the same
    else if (expr.getPath().equals(ref.getPath())) {
      if (exprContainsStar && exprHasPrefix) {
        assert exprPrefix != null;

        int k = 0;
        result.outputNames = Lists.newArrayListWithCapacity(incomingSchemaSize);
        for (int j=0; j < incomingSchemaSize; j++) {
          result.outputNames.add(EMPTY_STRING)// initialize
        }

        for(VectorWrapper<?> wrapper : incoming) {
          ValueVector vvIn = wrapper.getValueVector();
          String incomingName = vvIn.getField().getPath().getRootSegment().getPath();
          // get the prefix of the name
          String[] nameComponents = incomingName.split(StarColumnHelper.PREFIX_DELIMITER, 2);
          // if incoming valuevector does not have a prefix, ignore it since this expression is not referencing it
          if (nameComponents.length <= 1) {
            k++;
            continue;
         
          String namePrefix = nameComponents[0];
          if (exprPrefix.equals(namePrefix)) {
            String newName = incomingName;
            if (!result.outputMap.containsKey(newName)) {
              result.outputNames.set(k, newName);
              result.outputMap.put(newName,  newName);
            }
          }
          k++;
        }
      } else {
        result.outputNames = Lists.newArrayList();
        if (exprIsStar) {
          for (VectorWrapper<?> wrapper : incoming) {
            ValueVector vvIn = wrapper.getValueVector();
            String incomingName = vvIn.getField().getPath().getRootSegment().getPath();
            if (refIsStar) {
              addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project
            } else {
              addToResultMaps(incomingName, result, false);
            }
          }
        } else {
          String newName = expr.getPath();
          if (!refHasPrefix && !exprHasPrefix) {
            addToResultMaps(newName, result, true); // allow dups since this is likely top-level project
          } else {
            addToResultMaps(newName, result, false);
          }
        }
      }
    }
    // only the output has prefix
    else if (!exprHasPrefix && refHasPrefix) {
      result.outputNames = Lists.newArrayList();
      String newName = ref.getPath();
      addToResultMaps(newName, result, false);
    }
    // input has prefix but output does not; this would happen for a top-level project
    else if (exprHasPrefix && !refHasPrefix) {
      int k = 0;
      result.outputNames = Lists.newArrayListWithCapacity(incomingSchemaSize);
      for (int j=0; j < incomingSchemaSize; j++) {
        result.outputNames.add(EMPTY_STRING)// initialize
      }
      // remove the prefix from the incoming column names     
      for(VectorWrapper<?> wrapper : incoming) {
        ValueVector vvIn = wrapper.getValueVector();
        String name = vvIn.getField().getPath().getRootSegment().getPath();
        String[] components = name.split(StarColumnHelper.PREFIX_DELIMITER, 2);
        if (components.length <= 1)  {
          k++;
          continue;
       
View Full Code Here

      if (expr == null) continue;
      keyExprsBuild[i] = expr;
     
      final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
      // create a type-specific ValueVector for this key
      ValueVector vv = TypeHelper.getNewVector(outputField, allocator);
      vv.allocateNew();
      htKeyFieldIds[i] = htContainerOrig.add(vv);
     
      i++;
    }
View Full Code Here

      for (QueryResultBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper<?> vw : loader) {
          System.out.println(vw.getValueVector().getField().toExpr());
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            System.out.println(vv.getAccessor().getObject(i));
          }
        }
        loader.clear();
        b.release();
      }
View Full Code Here

      for (QueryResultBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper<?> vw : loader) {
          System.out.println(vw.getValueVector().getField().toExpr());
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            System.out.println(vv.getAccessor().getObject(i));
          }
        }
        loader.clear();
        b.release();
      }
View Full Code Here

      for (QueryResultBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper<?> vw : loader) {
          System.out.println(vw.getValueVector().getField().toExpr());
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            if (vv instanceof VarBinaryVector) {
              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
              VarBinaryHolder vbh = new VarBinaryHolder();
              x.get(i, vbh);
              System.out.printf("%d..%d", vbh.start, vbh.end);

              System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
            } else {
              System.out.println(vv.getAccessor().getObject(i));
            }

          }
        }
        loader.clear();
View Full Code Here

      for (QueryResultBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper vw : loader) {
          System.out.println(vw.getValueVector().getField().toExpr());
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            if (vv instanceof VarBinaryVector) {
              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
              VarBinaryHolder vbh = new VarBinaryHolder();
              x.get(i, vbh);
              System.out.printf("%d..%d", vbh.start, vbh.end);

              System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
            } else {
              System.out.println(vv.getAccessor().getObject(i));
            }

          }
        }
        loader.clear();
View Full Code Here

      for (QueryResultBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper vw : loader) {
          System.out.println(vw.getValueVector().getField().toExpr());
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            if (vv instanceof VarBinaryVector) {
              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
              VarBinaryHolder vbh = new VarBinaryHolder();
              x.get(i, vbh);
              System.out.printf("%d..%d", vbh.start, vbh.end);

              System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
            } else {
              System.out.println(vv.getAccessor().getObject(i));
            }

          }
        }
        loader.clear();
View Full Code Here

TOP

Related Classes of org.apache.drill.exec.vector.ValueVector

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.