Package org.apache.tajo

Examples of org.apache.tajo.ExecutionBlockId


  private Map<ExecutionBlockId, ExecutionBlock> execBlockMap = new HashMap<ExecutionBlockId, ExecutionBlock>();
  private SimpleDirectedGraph<ExecutionBlockId, DataChannel> execBlockGraph =
      new SimpleDirectedGraph<ExecutionBlockId, DataChannel>();

  public ExecutionBlockId newExecutionBlockId() {
    return new ExecutionBlockId(queryId, nextId.incrementAndGet());
  }
View Full Code Here


    this(srcId, targetId, shuffleType);
    this.numOutputs = numOutputs;
  }

  public DataChannel(DataChannelProto proto) {
    this.srcId = new ExecutionBlockId(proto.getSrcId());
    this.targetId = new ExecutionBlockId(proto.getTargetId());
    this.transmitType = proto.getTransmitType();
    this.shuffleType = proto.getShuffleType();
    if (proto.hasSchema()) {
      this.setSchema(new Schema(proto.getSchema()));
    }
View Full Code Here

    this.taskRunnerManager = taskRunnerManager;
    this.connPool = RpcConnectionPool.getPool(conf);
    this.fetchLauncher = Executors.newFixedThreadPool(
        conf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM));
    try {
      final ExecutionBlockId executionBlockId = TajoIdUtils.createExecutionBlockId(args[1]);

      LOG.info("Tajo Root Dir: " + conf.getVar(ConfVars.ROOT_DIR));
      LOG.info("Worker Local Dir: " + conf.getVar(ConfVars.WORKER_TEMPORAL_DIR));

      UserGroupInformation.setConfiguration(conf);

      // QueryBlockId from String
      // NodeId has a form of hostname:port.
      NodeId nodeId = ConverterUtils.toNodeId(args[2]);
      this.containerId = ConverterUtils.toContainerId(args[3]);

      // QueryMaster's address
      String host = args[4];
      int port = Integer.parseInt(args[5]);
      this.qmMasterAddr = NetUtils.createSocketAddrForHost(host, port);

      LOG.info("QueryMaster Address:" + qmMasterAddr);
      // TODO - 'load credential' should be implemented
      // Getting taskOwner
      UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(conf.getVar(ConfVars.USERNAME));
      //taskOwner.addToken(token);

      // initialize MasterWorkerProtocol as an actual task owner.
//      this.client =
//          taskOwner.doAs(new PrivilegedExceptionAction<AsyncRpcClient>() {
//            @Override
//            public AsyncRpcClient run() throws Exception {
//              return new AsyncRpcClient(TajoWorkerProtocol.class, masterAddr);
//            }
//          });
//      this.master = client.getStub();

      this.executionBlockId = executionBlockId;
      this.queryId = executionBlockId.getQueryId();
      this.nodeId = nodeId;
      this.taskOwner = taskOwner;

      this.taskRunnerContext = new TaskRunnerContext();
    } catch (Exception e) {
View Full Code Here

  @Test
  public void testCreateHashFetchURL() throws Exception {
    QueryId q1 = TestTajoIds.createQueryId(1315890136000l, 2);
    String hostName = "tajo1";
    int port = 1234;
    ExecutionBlockId sid = new ExecutionBlockId(q1, 2);
    int partitionId = 2;

    List<QueryUnit.IntermediateEntry> intermediateEntries = TUtil.newList();
    for (int i = 0; i < 1000; i++) {
      intermediateEntries.add(new QueryUnit.IntermediateEntry(i, 0, partitionId, hostName, port));
View Full Code Here

  }
 
  @Test
  public final void testInterDataRetriver() throws Exception {
    MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
    ExecutionBlockId schid = plan.newExecutionBlockId();
    QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
    QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);
   
    File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
    qid1Dir.mkdirs();
View Full Code Here

  }
 
  @Test(expected = FileNotFoundException.class)
  public final void testNoSuchFile() throws Exception {
    MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
    ExecutionBlockId schid = plan.newExecutionBlockId();
    QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
    QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);
   
    File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
    qid1Dir.mkdirs();
View Full Code Here

    final int minMemory = 256;
    final int maxMemory = 512;
    float diskSlots = 1.0f;

    QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 1);
    ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(queryId);

    WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
        .setResourceRequestPriority(ResourceRequestPriority.MEMORY)
        .setNumContainers(60)
        .setExecutionBlockId(ebId.getProto())
        .setMaxDiskSlotPerContainer(diskSlots)
        .setMinDiskSlotPerContainer(diskSlots)
        .setMinMemoryMBPerContainer(minMemory)
        .setMaxMemoryMBPerContainer(maxMemory)
        .build();
View Full Code Here

    final int minMemory = 200;
    final int maxMemory = 500;
    float diskSlots = 1.0f;

    QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 2);
    ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(queryId);

    int requiredContainers = 60;

    int numAllocatedContainers = 0;

    int loopCount = 0;
    while(true) {
      WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
          .setResourceRequestPriority(ResourceRequestPriority.MEMORY)
          .setNumContainers(requiredContainers - numAllocatedContainers)
          .setExecutionBlockId(ebId.getProto())
          .setMaxDiskSlotPerContainer(diskSlots)
          .setMinDiskSlotPerContainer(diskSlots)
          .setMinMemoryMBPerContainer(minMemory)
          .setMaxMemoryMBPerContainer(maxMemory)
          .build();
View Full Code Here

    final float minDiskSlots = 1.0f;
    final float maxDiskSlots = 2.0f;
    int memoryMB = 256;

    QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 3);
    ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(queryId);

    WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
        .setResourceRequestPriority(ResourceRequestPriority.DISK)
        .setNumContainers(60)
        .setExecutionBlockId(ebId.getProto())
        .setMaxDiskSlotPerContainer(maxDiskSlots)
        .setMinDiskSlotPerContainer(minDiskSlots)
        .setMinMemoryMBPerContainer(memoryMB)
        .setMaxMemoryMBPerContainer(memoryMB)
        .build();
View Full Code Here

      }
      Tuple tuple = new VTuple(datums);
      tupleData.add(tuple);
    }

    ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(
        QueryIdFactory.newQueryId(System.currentTimeMillis(), 0));

    TupleCacheKey cacheKey = new TupleCacheKey(ebId.toString(), "TestTable");
    TupleCache tupleCache = TupleCache.getInstance();

    assertFalse(tupleCache.isBroadcastCacheReady(cacheKey));
    assertTrue(tupleCache.lockBroadcastScan(cacheKey));
    assertFalse(tupleCache.lockBroadcastScan(cacheKey));
View Full Code Here

TOP

Related Classes of org.apache.tajo.ExecutionBlockId

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.