Package org.apache.drill.exec.memory

Examples of org.apache.drill.exec.memory.BufferAllocator


  @Test
  //cast to varbinary(length)
  public void testCastVarBinary(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();

    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      VarBinaryVector c0 = exec.getValueVectorById(new SchemaPath("int_lit_cast", ExpressionPosition.UNKNOWN), VarBinaryVector.class);
      VarBinaryVector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          VarBinaryHolder holder0 = new VarBinaryHolder();
          a0.get(i, holder0);
          assertEquals("123", StringFunctionHelpers.toStringFromUTF8(holder0.start, holder0.end, holder0.buffer));
          ++count;

      }
      assertEquals(5, count);
    }
    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
    assertTrue(!context.isFailed());
View Full Code Here


  @Test
  //nested: cast is nested in another cast, or another function.
  public void testCastNested(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();
    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
      IntVector c0 = exec.getValueVectorById(new SchemaPath("add_cast", ExpressionPosition.UNKNOWN),IntVector.class);
      IntVector.Accessor a0;
      a0 = c0.getAccessor();

      int count = 0;
      for(int i = 0; i < c0.getAccessor().getValueCount(); i++){
          IntHolder holder0 = new IntHolder();
          a0.get(i, holder0);
          assertEquals(300, holder0.value);
          ++count;

      }
      assertEquals(5, count);
    }
    exec.stop();

    context.close();
    allocator.close();

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
    }
View Full Code Here

  @Test(expected = NumberFormatException.class)
  public void testCastNumException(@Injectable final DrillbitContext bitContext,
                            @Injectable UserServer.UserClientConnection connection) throws Throwable{

    final BufferAllocator allocator = new TopLevelAllocator();

    new NonStrictExpectations(){{
      bitContext.getMetrics(); result = new MetricRegistry();
      bitContext.getAllocator(); result = allocator;
      bitContext.getConfig(); result = c;
      bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
      bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    }};

    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8));
    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

    while(exec.next()){
    }

    exec.stop();

    context.close();
    allocator.close();

    assertTrue(context.isFailed());

    if(context.getFailureCause() != null){
      throw context.getFailureCause();
View Full Code Here

      if (manager == null) {
        if (body != null) {
          body.release();
        }
      }
      BufferAllocator allocator = manager.getFragmentContext().getAllocator();
      if (body != null) {
        if (!allocator.takeOwnership((DrillBuf) body.unwrap())) {
          dataHandler.handle(connection, manager, OOM_FRAGMENT, null, null);
        }
      }
      dataHandler.handle(connection, manager, fragmentBatch, (DrillBuf) body, sender);
View Full Code Here

public class TestAdaptiveAllocation {

  @Test
  public void test() throws Exception {
    BufferAllocator allocator = new TopLevelAllocator();
    MaterializedField field = MaterializedField.create("field", Types.required(MinorType.VARCHAR));
    NullableVarBinaryVector vector1 = new NullableVarBinaryVector(field, allocator);
    NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator);
    NullableBigIntVector vector3 = new NullableBigIntVector(field, allocator);
View Full Code Here

  @Consumes(MediaType.APPLICATION_JSON)
  @Produces(MediaType.APPLICATION_JSON)
  public List<Map<String, Object>> submitQueryJSON(QueryWrapper query) throws Exception {
    final DrillConfig config = work.getContext().getConfig();
    final ClusterCoordinator coordinator = work.getContext().getClusterCoordinator();
    final BufferAllocator allocator = work.getContext().getAllocator();
    return query.run(config, coordinator, allocator);
  }
View Full Code Here

public class TestLoad extends ExecTest {

  @Test
  public void testLoadValueVector() throws Exception {
    BufferAllocator allocator = new TopLevelAllocator();
    ValueVector fixedV = new IntVector(MaterializedField.create(new SchemaPath("ints", ExpressionPosition.UNKNOWN),
        Types.required(MinorType.INT)), allocator);
    ValueVector varlenV = new VarCharVector(MaterializedField.create(
        new SchemaPath("chars", ExpressionPosition.UNKNOWN), Types.required(MinorType.VARCHAR)), allocator);
    ValueVector nullableVarlenV = new NullableVarCharVector(MaterializedField.create(new SchemaPath("chars",
        ExpressionPosition.UNKNOWN), Types.optional(MinorType.VARCHAR)), allocator);

    List<ValueVector> vectors = Lists.newArrayList(fixedV, varlenV, nullableVarlenV);
    for (ValueVector v : vectors) {
      AllocationHelper.allocate(v, 100, 50);
      v.getMutator().generateTestData(100);
    }

    WritableBatch writableBatch = WritableBatch.getBatchNoHV(100, vectors, false);
    RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    ByteBuf[] byteBufs = writableBatch.getBuffers();
    int bytes = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      bytes += byteBufs[i].writerIndex();
    }
    DrillBuf byteBuf = allocator.buffer(bytes);
    int index = 0;
    for (int i = 0; i < byteBufs.length; i++) {
      byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex());
      index += byteBufs[i].writerIndex();
    }
View Full Code Here

    columns.add(new SchemaPath("_MAP.bin", ExpressionPosition.UNKNOWN));
    columns.add(new SchemaPath("_MAP.bin2", ExpressionPosition.UNKNOWN));
    int totalRowCount = 0;

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++) {
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();

      int rowCount = 0;
      while ((rowCount = rr.next()) > 0) {
        totalRowCount += rowCount;
      }
      System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
      rr.cleanup();
    }

    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
  }
View Full Code Here

      if (manager == null) {
        if (body != null) {
          body.release();
        }
      }
      BufferAllocator allocator = manager.getFragmentContext().getAllocator();
      if(body != null){
        if(!allocator.takeOwnership((DrillBuf) body.unwrap())){
          dataHandler.handle(connection, manager, OOM_FRAGMENT, null, null);
        }
      }
      dataHandler.handle(connection, manager, fragmentBatch, (DrillBuf) body, sender);
View Full Code Here

TOP

Related Classes of org.apache.drill.exec.memory.BufferAllocator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.