Package org.apache.tajo.engine.planner.global

Examples of org.apache.tajo.engine.planner.global.ExecutionBlock


  private static int HTTP_REQUEST_MAXIMUM_LENGTH = 1900;

  public static QueryUnit[] createJoinTasks(SubQuery subQuery)
      throws IOException {
    MasterPlan masterPlan = subQuery.getMasterPlan();
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryMasterTask.QueryMasterTaskContext masterContext = subQuery.getContext();
    AbstractStorageManager storageManager = subQuery.getStorageManager();

    ScanNode[] scans = execBlock.getScanNodes();

    Path tablePath;
    Fragment [] fragments = new Fragment[2];
    TableStat [] stats = new TableStat[2];

    // initialize variables from the child operators
    for (int i =0; i < 2; i++) {
      TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName());
      if (tableDesc == null) { // if it is a real table stored on storage
        // TODO - to be fixed (wrong directory)
        ExecutionBlock [] childBlocks = new ExecutionBlock[2];
        childBlocks[0] = masterPlan.getChild(execBlock.getId(), 0);
        childBlocks[1] = masterPlan.getChild(execBlock.getId(), 1);

        tablePath = storageManager.getTablePath(scans[i].getTableName());
        stats[i] = masterContext.getSubQuery(childBlocks[i].getId()).getTableStat();
        fragments[i] = new Fragment(scans[i].getCanonicalName(), tablePath,
            CatalogUtil.newTableMeta(scans[i].getInSchema(), StoreType.CSV), 0, 0);
      } else {
        tablePath = tableDesc.getPath();
        stats[i] = tableDesc.getMeta().getStat();
        fragments[i] = storageManager.getSplits(scans[i].getCanonicalName(),
            tableDesc.getMeta(), tablePath).get(0);
      }
    }

    // Assigning either fragments or fetch urls to query units
    QueryUnit [] tasks;
    boolean leftSmall = execBlock.isBroadcastTable(scans[0].getCanonicalName());
    boolean rightSmall = execBlock.isBroadcastTable(scans[1].getCanonicalName());

    if (leftSmall && rightSmall) {
      LOG.info("[Distributed Join Strategy] : Immediate Two Way Join on Single Machine");
      tasks = new QueryUnit[1];
      tasks[0] = new QueryUnit(QueryIdFactory.newQueryUnitId(subQuery.getId(), 0),
          false, subQuery.getEventHandler());
      tasks[0].setLogicalPlan(execBlock.getPlan());
      tasks[0].setFragment(scans[0].getCanonicalName(), fragments[0]);
      tasks[0].setFragment(scans[1].getCanonicalName(), fragments[1]);
    } else if (leftSmall ^ rightSmall) {
      LOG.info("[Distributed Join Strategy] : Broadcast Join");
      int broadcastIdx = leftSmall ? 0 : 1;
View Full Code Here


    return tasks;
  }

  private static QueryUnit [] createLeafTasksWithBroadcastTable(SubQuery subQuery, int baseScanId, Fragment broadcasted) throws IOException {
    ExecutionBlock execBlock = subQuery.getBlock();
    ScanNode[] scans = execBlock.getScanNodes();
    Preconditions.checkArgument(scans.length == 2, "Must be Join Query");
    TableMeta meta;
    Path inputPath;
    ScanNode scan = scans[baseScanId];
    TableDesc desc = subQuery.getContext().getTableDescMap().get(scan.getCanonicalName());
View Full Code Here

    }
    return queryUnits.toArray(new QueryUnit[queryUnits.size()]);
  }

  private static QueryUnit newQueryUnit(SubQuery subQuery, int taskId, Fragment fragment) {
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryUnit unit = new QueryUnit(
        QueryIdFactory.newQueryUnitId(subQuery.getId(), taskId), execBlock.isLeafBlock(),
        subQuery.getEventHandler());
    unit.setLogicalPlan(execBlock.getPlan());
    unit.setFragment2(fragment);
    return unit;
  }
View Full Code Here

    unit.setFragment2(fragment);
    return unit;
  }

  private static QueryUnit [] newEmptyJoinTask(SubQuery subQuery, Fragment [] fragments, int taskNum) {
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryUnit [] tasks = new QueryUnit[taskNum];
    for (int i = 0; i < taskNum; i++) {
      tasks[i] = new QueryUnit(
          QueryIdFactory.newQueryUnitId(subQuery.getId(), i), execBlock.isLeafBlock(),
          subQuery.getEventHandler());
      tasks[i].setLogicalPlan(execBlock.getPlan());
      for (Fragment fragment : fragments) {
        tasks[i].setFragment2(fragment);
      }
    }
View Full Code Here

  }

  public static QueryUnit [] createRangePartitionedTasks(SubQuery subQuery,
                                                         SubQuery childSubQuery, DataChannel channel, int maxNum)
      throws InternalException {
    ExecutionBlock execBlock = subQuery.getBlock();
    TableStat stat = childSubQuery.getTableStat();
    if (stat.getNumRows() == 0) {
      return new QueryUnit[0];
    }

    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());

    SortNode sortNode = PlannerUtil.findTopNode(childSubQuery.getBlock().getPlan(), NodeType.SORT);
    SortSpec [] sortSpecs = sortNode.getSortKeys();
View Full Code Here

    return sb.toString();
  }

  public static QueryUnit [] createHashPartitionedTasks(MasterPlan masterPlan, SubQuery subQuery,
                                                 SubQuery childSubQuery, DataChannel channel, int maxNum) {
    ExecutionBlock execBlock = subQuery.getBlock();

    List<TableStat> tableStats = new ArrayList<TableStat>();
    List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
    for (ExecutionBlock childBlock : childBlocks) {
      SubQuery childExecSM = subQuery.getContext().getSubQuery(childBlock.getId());
      tableStats.add(childExecSM.getTableStat());
    }
    TableStat totalStat = StatisticsUtil.computeStatFromUnionBlock(tableStats);

    if (totalStat.getNumRows() == 0) {
      return new QueryUnit[0];
    }

    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());


    Fragment frag = new Fragment(scan.getCanonicalName(), tablePath,
View Full Code Here

    return hashed;
  }

  public static SubQuery setPartitionNumberForTwoPhase(SubQuery subQuery, final int desiredNum, DataChannel channel) {
    ExecutionBlock execBlock = subQuery.getBlock();
    Column[] keys = null;
    // if the next query is join,
    // set the partition number for the current logicalUnit
    // TODO: the union handling is required when a join has unions as its child
    MasterPlan masterPlan = subQuery.getMasterPlan();
    keys = channel.getPartitionKey();
    if (!masterPlan.isRoot(subQuery.getBlock()) ) {
      ExecutionBlock parentBlock = masterPlan.getParent(subQuery.getBlock());
      if (parentBlock.getPlan().getType() == NodeType.JOIN) {
        channel.setPartitionNum(desiredNum);
      }
    }

View Full Code Here

      SubQueryCompletedEvent castEvent = (SubQueryCompletedEvent) event;
      ExecutionBlockCursor cursor = query.getExecutionBlockCursor();
      MasterPlan masterPlan = query.getPlan();
      // if the subquery is succeeded
      if (castEvent.getFinalState() == SubQueryState.SUCCEEDED) {
        ExecutionBlock nextBlock = cursor.nextBlock();
        if (!query.getPlan().isTerminal(nextBlock) || !query.getPlan().isRoot(nextBlock)) {
          SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock, query.sm);
          nextSubQuery.setPriority(query.priority--);
          query.addSubQuery(nextSubQuery);
          nextSubQuery.handle(new SubQueryEvent(nextSubQuery.getId(),
              SubQueryEventType.SQ_INIT));
          LOG.info("Scheduling SubQuery:" + nextSubQuery.getId());
          if(LOG.isDebugEnabled()) {
            LOG.debug("Scheduling SubQuery's Priority: " + nextSubQuery.getPriority());
            LOG.debug("Scheduling SubQuery's Plan: \n" + nextSubQuery.getBlock().getPlan());
          }
          return query.checkQueryForCompleted();

        } else { // Finish a query
          if (query.checkQueryForCompleted() == QueryState.QUERY_SUCCEEDED) {
            DataChannel finalChannel = masterPlan.getChannel(castEvent.getExecutionBlockId(), nextBlock.getId());
            Path finalOutputDir = commitOutputData(query);
            TableDesc finalTableDesc = buildOrUpdateResultTableDesc(query, castEvent.getExecutionBlockId(), finalOutputDir);

            QueryContext queryContext = query.context.getQueryContext();
            CatalogService catalog = query.context.getQueryMasterContext().getWorkerContext().getCatalog();
View Full Code Here

      SubQueryEvent, SubQueryState> {

    @Override
    public SubQueryState transition(SubQuery subQuery, SubQueryEvent subQueryEvent) {
      subQuery.setStartTime();
      ExecutionBlock execBlock = subQuery.getBlock();
      SubQueryState state;

      try {
        // Union operator does not require actual query processing. It is performed logically.
        if (execBlock.hasUnion()) {
          subQuery.finish();
          state = SubQueryState.SUCCEEDED;
        } else {
          ExecutionBlock parent = subQuery.getMasterPlan().getParent(subQuery.getBlock());
          DataChannel channel = subQuery.getMasterPlan().getChannel(subQuery.getId(), parent.getId());
          setRepartitionIfNecessary(subQuery, channel);
          createTasks(subQuery);

          if (subQuery.tasks.size() == 0) { // if there is no tasks
            subQuery.finish();
View Full Code Here

     * @return
     */
    public static int calculatePartitionNum(SubQuery subQuery, DataChannel channel) {
      TajoConf conf = subQuery.context.getConf();
      MasterPlan masterPlan = subQuery.getMasterPlan();
      ExecutionBlock parent = masterPlan.getParent(subQuery.getBlock());

      GroupbyNode grpNode = null;
      if (parent != null) {
        grpNode = PlannerUtil.findTopNode(parent.getPlan(), NodeType.GROUP_BY);
      }

      // Is this subquery the first step of join?
      if (parent != null && parent.getScanNodes().length == 2) {
        List<ExecutionBlock> childs = masterPlan.getChilds(parent);

        // for inner
        ExecutionBlock outer = childs.get(0);
        long outerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, outer);

        // for inner
        ExecutionBlock inner = childs.get(1);
        long innerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, inner);
        LOG.info("Outer volume: " + Math.ceil((double)outerVolume / 1048576));
        LOG.info("Inner volume: " + Math.ceil((double)innerVolume / 1048576));

        long smaller = Math.min(outerVolume, innerVolume);
View Full Code Here

TOP

Related Classes of org.apache.tajo.engine.planner.global.ExecutionBlock

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.