Examples of ExecutionBlock

org.apache.tajo.engine.planner.global.ExecutionBlock
A distributed execution plan (DEP) is a direct acyclic graph (DAG) of ExecutionBlocks. An ExecutionBlock is a basic execution unit that could be distributed across a number of nodes. An ExecutionBlock class contains input information (e.g., child execution blocks or input tables), and output information (e.g., partition type, partition key, and partition number). In addition, it includes a logical plan to be executed in each node.

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

  private static int HTTP_REQUEST_MAXIMUM_LENGTH = 1900;


  public static QueryUnit[] createJoinTasks(SubQuery subQuery)
      throws IOException {
    MasterPlan masterPlan = subQuery.getMasterPlan();
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryMasterTask.QueryMasterTaskContext masterContext = subQuery.getContext();
    AbstractStorageManager storageManager = subQuery.getStorageManager();


    ScanNode[] scans = execBlock.getScanNodes();


    Path tablePath;
    Fragment [] fragments = new Fragment[2];
    TableStat [] stats = new TableStat[2];


    // initialize variables from the child operators
    for (int i =0; i < 2; i++) {
      TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName());
      if (tableDesc == null) { // if it is a real table stored on storage
        // TODO - to be fixed (wrong directory)
        ExecutionBlock [] childBlocks = new ExecutionBlock[2];
        childBlocks[0] = masterPlan.getChild(execBlock.getId(), 0);
        childBlocks[1] = masterPlan.getChild(execBlock.getId(), 1);


        tablePath = storageManager.getTablePath(scans[i].getTableName());
        stats[i] = masterContext.getSubQuery(childBlocks[i].getId()).getTableStat();
        fragments[i] = new Fragment(scans[i].getCanonicalName(), tablePath,
            CatalogUtil.newTableMeta(scans[i].getInSchema(), StoreType.CSV), 0, 0);
      } else {
        tablePath = tableDesc.getPath();
        stats[i] = tableDesc.getMeta().getStat();
        fragments[i] = storageManager.getSplits(scans[i].getCanonicalName(),
            tableDesc.getMeta(), tablePath).get(0);
      }
    }


    // Assigning either fragments or fetch urls to query units
    QueryUnit [] tasks;
    boolean leftSmall = execBlock.isBroadcastTable(scans[0].getCanonicalName());
    boolean rightSmall = execBlock.isBroadcastTable(scans[1].getCanonicalName());


    if (leftSmall && rightSmall) {
      LOG.info("[Distributed Join Strategy] : Immediate Two Way Join on Single Machine");
      tasks = new QueryUnit[1];
      tasks[0] = new QueryUnit(QueryIdFactory.newQueryUnitId(subQuery.getId(), 0),
          false, subQuery.getEventHandler());
      tasks[0].setLogicalPlan(execBlock.getPlan());
      tasks[0].setFragment(scans[0].getCanonicalName(), fragments[0]);
      tasks[0].setFragment(scans[1].getCanonicalName(), fragments[1]);
    } else if (leftSmall ^ rightSmall) {
      LOG.info("[Distributed Join Strategy] : Broadcast Join");
      int broadcastIdx = leftSmall ? 0 : 1;

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock


    return tasks;
  }


  private static QueryUnit [] createLeafTasksWithBroadcastTable(SubQuery subQuery, int baseScanId, Fragment broadcasted) throws IOException {
    ExecutionBlock execBlock = subQuery.getBlock();
    ScanNode[] scans = execBlock.getScanNodes();
    Preconditions.checkArgument(scans.length == 2, "Must be Join Query");
    TableMeta meta;
    Path inputPath;
    ScanNode scan = scans[baseScanId];
    TableDesc desc = subQuery.getContext().getTableDescMap().get(scan.getCanonicalName());

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

    }
    return queryUnits.toArray(new QueryUnit[queryUnits.size()]);
  }


  private static QueryUnit newQueryUnit(SubQuery subQuery, int taskId, Fragment fragment) {
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryUnit unit = new QueryUnit(
        QueryIdFactory.newQueryUnitId(subQuery.getId(), taskId), execBlock.isLeafBlock(),
        subQuery.getEventHandler());
    unit.setLogicalPlan(execBlock.getPlan());
    unit.setFragment2(fragment);
    return unit;
  }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

    unit.setFragment2(fragment);
    return unit;
  }


  private static QueryUnit [] newEmptyJoinTask(SubQuery subQuery, Fragment [] fragments, int taskNum) {
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryUnit [] tasks = new QueryUnit[taskNum];
    for (int i = 0; i < taskNum; i++) {
      tasks[i] = new QueryUnit(
          QueryIdFactory.newQueryUnitId(subQuery.getId(), i), execBlock.isLeafBlock(),
          subQuery.getEventHandler());
      tasks[i].setLogicalPlan(execBlock.getPlan());
      for (Fragment fragment : fragments) {
        tasks[i].setFragment2(fragment);
      }
    }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

  }


  public static QueryUnit [] createRangePartitionedTasks(SubQuery subQuery,
                                                         SubQuery childSubQuery, DataChannel channel, int maxNum)
      throws InternalException {
    ExecutionBlock execBlock = subQuery.getBlock();
    TableStat stat = childSubQuery.getTableStat();
    if (stat.getNumRows() == 0) {
      return new QueryUnit[0];
    }


    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());


    SortNode sortNode = PlannerUtil.findTopNode(childSubQuery.getBlock().getPlan(), NodeType.SORT);
    SortSpec [] sortSpecs = sortNode.getSortKeys();

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

    return sb.toString();
  }


  public static QueryUnit [] createHashPartitionedTasks(MasterPlan masterPlan, SubQuery subQuery,
                                                 SubQuery childSubQuery, DataChannel channel, int maxNum) {
    ExecutionBlock execBlock = subQuery.getBlock();


    List<TableStat> tableStats = new ArrayList<TableStat>();
    List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
    for (ExecutionBlock childBlock : childBlocks) {
      SubQuery childExecSM = subQuery.getContext().getSubQuery(childBlock.getId());
      tableStats.add(childExecSM.getTableStat());
    }
    TableStat totalStat = StatisticsUtil.computeStatFromUnionBlock(tableStats);


    if (totalStat.getNumRows() == 0) {
      return new QueryUnit[0];
    }


    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());




    Fragment frag = new Fragment(scan.getCanonicalName(), tablePath,

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock


    return hashed;
  }


  public static SubQuery setPartitionNumberForTwoPhase(SubQuery subQuery, final int desiredNum, DataChannel channel) {
    ExecutionBlock execBlock = subQuery.getBlock();
    Column[] keys = null;
    // if the next query is join,
    // set the partition number for the current logicalUnit
    // TODO: the union handling is required when a join has unions as its child
    MasterPlan masterPlan = subQuery.getMasterPlan();
    keys = channel.getPartitionKey();
    if (!masterPlan.isRoot(subQuery.getBlock()) ) {
      ExecutionBlock parentBlock = masterPlan.getParent(subQuery.getBlock());
      if (parentBlock.getPlan().getType() == NodeType.JOIN) {
        channel.setPartitionNum(desiredNum);
      }
    }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

      SubQueryCompletedEvent castEvent = (SubQueryCompletedEvent) event;
      ExecutionBlockCursor cursor = query.getExecutionBlockCursor();
      MasterPlan masterPlan = query.getPlan();
      // if the subquery is succeeded
      if (castEvent.getFinalState() == SubQueryState.SUCCEEDED) {
        ExecutionBlock nextBlock = cursor.nextBlock();
        if (!query.getPlan().isTerminal(nextBlock) || !query.getPlan().isRoot(nextBlock)) {
          SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock, query.sm);
          nextSubQuery.setPriority(query.priority--);
          query.addSubQuery(nextSubQuery);
          nextSubQuery.handle(new SubQueryEvent(nextSubQuery.getId(),
              SubQueryEventType.SQ_INIT));
          LOG.info("Scheduling SubQuery:" + nextSubQuery.getId());
          if(LOG.isDebugEnabled()) {
            LOG.debug("Scheduling SubQuery's Priority: " + nextSubQuery.getPriority());
            LOG.debug("Scheduling SubQuery's Plan: \n" + nextSubQuery.getBlock().getPlan());
          }
          return query.checkQueryForCompleted();


        } else { // Finish a query
          if (query.checkQueryForCompleted() == QueryState.QUERY_SUCCEEDED) {
            DataChannel finalChannel = masterPlan.getChannel(castEvent.getExecutionBlockId(), nextBlock.getId());
            Path finalOutputDir = commitOutputData(query);
            TableDesc finalTableDesc = buildOrUpdateResultTableDesc(query, castEvent.getExecutionBlockId(), finalOutputDir);


            QueryContext queryContext = query.context.getQueryContext();
            CatalogService catalog = query.context.getQueryMasterContext().getWorkerContext().getCatalog();

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

      SubQueryEvent, SubQueryState> {


    @Override
    public SubQueryState transition(SubQuery subQuery, SubQueryEvent subQueryEvent) {
      subQuery.setStartTime();
      ExecutionBlock execBlock = subQuery.getBlock();
      SubQueryState state;


      try {
        // Union operator does not require actual query processing. It is performed logically.
        if (execBlock.hasUnion()) {
          subQuery.finish();
          state = SubQueryState.SUCCEEDED;
        } else {
          ExecutionBlock parent = subQuery.getMasterPlan().getParent(subQuery.getBlock());
          DataChannel channel = subQuery.getMasterPlan().getChannel(subQuery.getId(), parent.getId());
          setRepartitionIfNecessary(subQuery, channel);
          createTasks(subQuery);


          if (subQuery.tasks.size() == 0) { // if there is no tasks
            subQuery.finish();

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock

     * @return
     */
    public static int calculatePartitionNum(SubQuery subQuery, DataChannel channel) {
      TajoConf conf = subQuery.context.getConf();
      MasterPlan masterPlan = subQuery.getMasterPlan();
      ExecutionBlock parent = masterPlan.getParent(subQuery.getBlock());


      GroupbyNode grpNode = null;
      if (parent != null) {
        grpNode = PlannerUtil.findTopNode(parent.getPlan(), NodeType.GROUP_BY);
      }


      // Is this subquery the first step of join?
      if (parent != null && parent.getScanNodes().length == 2) {
        List<ExecutionBlock> childs = masterPlan.getChilds(parent);


        // for inner
        ExecutionBlock outer = childs.get(0);
        long outerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, outer);


        // for inner
        ExecutionBlock inner = childs.get(1);
        long innerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, inner);
        LOG.info("Outer volume: " + Math.ceil((double)outerVolume / 1048576));
        LOG.info("Inner volume: " + Math.ceil((double)innerVolume / 1048576));


        long smaller = Math.min(outerVolume, innerVolume);

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.