Package org.apache.drill.exec.memory

Examples of org.apache.drill.exec.memory.BufferAllocator$PreAllocator


*/

public class TestLoad {
  @Test
  public void testLoadValueVector() {
    BufferAllocator allocator = BufferAllocator.getAllocator(null);
    ValueVector fixedV = new IntVector(
      MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN), Types.required(MinorType.INT)),
      allocator);
    ValueVector varlenV = new VarCharVector(
      MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)),
      allocator
    );
    ValueVector nullableVarlenV = new NullableVarCharVector(
      MaterializedField.create(new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)),
      allocator
    );

    List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV);
    for (ValueVector v : vectors) {
      AllocationHelper.allocate(v, 100, 50);
      v.getMutator().generateTestData();
      v.getMutator().setValueCount(100);
    }

    WritableBatch writableBatch = WritableBatch.getBatchNoSV(100, vectors);
    RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    ByteBuf[] byteBufs = writableBatch.getBuffers();
    int bytes = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      bytes += byteBufs[i].writerIndex();
    }
    ByteBuf byteBuf = allocator.buffer(bytes);
    int index = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
      index += byteBufs[i].writerIndex();
    }
View Full Code Here


public class TestAdaptiveAllocation {

  @Test
  public void test() throws Exception {
    BufferAllocator allocator = new TopLevelAllocator();
    MaterializedField field = MaterializedField.create("field", Types.required(MinorType.VARCHAR));
    NullableVarBinaryVector vector1 = new NullableVarBinaryVector(field, allocator);
    NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator);
    NullableBigIntVector vector3 = new NullableBigIntVector(field, allocator);
View Full Code Here

public class TestLoad extends ExecTest {

  @Test
  public void testLoadValueVector() throws Exception {
    BufferAllocator allocator = new TopLevelAllocator();
    ValueVector fixedV = new IntVector(MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN),
        Types.required(MinorType.INT)), allocator);
    ValueVector varlenV = new VarCharVector(MaterializedField.create(
        new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)), allocator);
    ValueVector nullableVarlenV = new NullableVarCharVector(MaterializedField.create(new SchemaPath("chars",
        ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)), allocator);

    List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV);
    for (ValueVector v : vectors) {
      AllocationHelper.allocate(v, 100, 50);
      v.getMutator().generateTestData(100);
    }

    WritableBatch writableBatch = WritableBatch.getBatchNoHV(100, vectors, false);
    RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    ByteBuf[] byteBufs = writableBatch.getBuffers();
    int bytes = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      bytes += byteBufs[i].writerIndex();
    }
    ByteBuf byteBuf = allocator.buffer(bytes);
    int index = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
      index += byteBufs[i].writerIndex();
    }
View Full Code Here

  @Consumes(MediaType.APPLICATION_JSON)
  @Produces(MediaType.APPLICATION_JSON)
  public List<Map<String, Object>> submitQueryJSON(QueryWrapper query) throws Exception {
    final DrillConfig config = work.getContext().getConfig();
    final ClusterCoordinator coordinator = work.getContext().getClusterCoordinator();
    final BufferAllocator allocator = work.getContext().getAllocator();
    return query.run(config, coordinator, allocator);
  }
View Full Code Here

      if (manager == null) {
        if (body != null) {
          body.release();
        }
      }
      BufferAllocator allocator = manager.getFragmentContext().getAllocator();
      if(body != null){
        if(!allocator.takeOwnership((AccountingByteBuf) body.unwrap())){
          dataHandler.handle(connection, manager, OOM_FRAGMENT, null, null);
        }
      }
      dataHandler.handle(connection, manager, fragmentBatch, body, sender);
View Full Code Here

public class TestSplitAndTransfer {

  @Test
  public void test() throws Exception {
    BufferAllocator allocator = new TopLevelAllocator();
    MaterializedField field = MaterializedField.create("field", Types.optional(MinorType.VARCHAR));
    NullableVarCharVector varCharVector = new NullableVarCharVector(field, allocator);
    varCharVector.allocateNew(10000, 1000);

    String[] compareArray = new String[500];

    for (int i = 0; i < 500; i += 3) {
      String s = String.format("%010d", i);
      varCharVector.getMutator().set(i, s.getBytes());
      compareArray[i] = s;
    }
    varCharVector.getMutator().setValueCount(500);

    TransferPair tp = varCharVector.getTransferPair();
    NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
    Accessor accessor = newVarCharVector.getAccessor();
    int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};

    for (int[] startLength : startLengths) {
      int start = startLength[0];
      int length = startLength[1];
      tp.splitAndTransfer(start, length);
      newVarCharVector.getMutator().setValueCount(length);
      for (int i = 0; i < length; i++) {
        boolean expectedSet = ((start + i) % 3) == 0;
        if (expectedSet) {
          byte[] expectedValue = compareArray[start + i].getBytes();
          Assert.assertFalse(accessor.isNull(i));
//          System.out.println(new String(accessor.get(i)));
          Assert.assertArrayEquals(expectedValue, accessor.get(i));
        } else {
          Assert.assertTrue(accessor.isNull(i));
        }
      }
      newVarCharVector.clear();
    }

    varCharVector.clear();
    allocator.close();
  }
View Full Code Here

  @Test
  //cast to int
  public void testCastInt(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();

    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      IntVector c0 = exec.getValueVectorById(new SchemaPath("varchar_cast", ExpressionPosition.UNKNOWN), IntVector.class);
      IntVector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          IntHolder holder0 = new IntHolder();
          a0.get(i, holder0);
          assertEquals(1256, holder0.value);
          ++count;

      }
      assertEquals(5, count);
    }

    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
View Full Code Here

  @Test
  //cast to float4
  public void testCastFloat4(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{
    final BufferAllocator allocator = new TopLevelAllocator();
    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      Float4Vector c0 = exec.getValueVectorById(new SchemaPath("varchar_cast2", ExpressionPosition.UNKNOWN), Float4Vector.class);
      Float4Vector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          Float4Holder holder0 = new Float4Holder();
          a0.get(i, holder0);
          assertEquals(12.56, holder0.value, 0.001);
          ++count;

      }
      assertEquals(5, count);
    }

    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
View Full Code Here

  @Test
  //cast to float8
  public void testCastFloat8(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();
    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      Float8Vector c0 = exec.getValueVectorById(new SchemaPath("varchar_cast2", ExpressionPosition.UNKNOWN), Float8Vector.class);
      Float8Vector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          Float8Holder holder0 = new Float8Holder();
          a0.get(i, holder0);
          assertEquals(12.56, holder0.value, 0.001);
          ++count;

      }
      assertEquals(5, count);
    }

    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
View Full Code Here

  @Test
  //cast to varchar(length)
  public void testCastVarChar(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();
    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      VarCharVector c0 = exec.getValueVectorById(new SchemaPath("int_lit_cast", ExpressionPosition.UNKNOWN), VarCharVector.class);
      VarCharVector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          VarCharHolder holder0 = new VarCharHolder();
          a0.get(i, holder0);
          assertEquals("123", holder0.toString());
          ++count;

      }
      assertEquals(5, count);
    }

    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
View Full Code Here

TOP

Related Classes of org.apache.drill.exec.memory.BufferAllocator$PreAllocator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.