Package org.apache.tajo.engine.planner.global

Examples of org.apache.tajo.engine.planner.global.DataChannel


      return dataChannel;
    }
    if (!p.hasDataChannel()) {
      return null;
    }
    this.dataChannel = new DataChannel(p.getDataChannel());
    return this.dataChannel;
  }
View Full Code Here


          }
          return query.checkQueryForCompleted();

        } else { // Finish a query
          if (query.checkQueryForCompleted() == QueryState.QUERY_SUCCEEDED) {
            DataChannel finalChannel = masterPlan.getChannel(castEvent.getExecutionBlockId(), nextBlock.getId());
            Path finalOutputDir = commitOutputData(query);
            TableDesc finalTableDesc = buildOrUpdateResultTableDesc(query, castEvent.getExecutionBlockId(), finalOutputDir);

            QueryContext queryContext = query.context.getQueryContext();
            CatalogService catalog = query.context.getQueryMasterContext().getWorkerContext().getCatalog();
View Full Code Here

    }
  }

  private PhysicalExec buildOutputOperator(TaskAttemptContext context, LogicalNode plan,
                                           PhysicalExec execPlan) throws IOException {
    DataChannel channel = context.getDataChannel();
    StoreTableNode storeTableNode = new StoreTableNode(UNGENERATED_PID, channel.getTargetId().toString());
    storeTableNode.setStorageType(CatalogProtos.StoreType.CSV);
    storeTableNode.setInSchema(plan.getOutSchema());
    storeTableNode.setOutSchema(plan.getOutSchema());
    if (channel.getPartitionType() != PartitionType.NONE_PARTITION) {
      storeTableNode.setPartitions(channel.getPartitionType(), channel.getPartitionKey(), channel.getPartitionNum());
    } else {
      storeTableNode.setDefaultParition();
    }
    storeTableNode.setChild(plan);
View Full Code Here

        if (execBlock.hasUnion()) {
          subQuery.finish();
          state = SubQueryState.SUCCEEDED;
        } else {
          ExecutionBlock parent = subQuery.getMasterPlan().getParent(subQuery.getBlock());
          DataChannel channel = subQuery.getMasterPlan().getChannel(subQuery.getId(), parent.getId());
          setRepartitionIfNecessary(subQuery, channel);
          createTasks(subQuery);

          if (subQuery.tasks.size() == 0) { // if there is no tasks
            subQuery.finish();
View Full Code Here

      } else { // Case 3: Others (Sort or Aggregation)
        int numTasks = getNonLeafTaskNum(subQuery);
        ExecutionBlockId childId = masterPlan.getChilds(subQuery.getBlock()).get(0).getId();
        SubQuery child = subQuery.context.getSubQuery(childId);
        DataChannel channel = masterPlan.getChannel(child.getId(), subQuery.getId());
        tasks = Repartitioner.createNonLeafTask(masterPlan, subQuery, child, channel, numTasks);
      }

      LOG.info("Create " + tasks.length + " Tasks");
View Full Code Here

      stat = computeStatFromUnionBlock(this);
    } else {
      stat = computeStatFromTasks();
    }

    DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0);
    // get default or store type
    CatalogProtos.StoreType storeType = CatalogProtos.StoreType.CSV; // default setting

    // if store plan (i.e., CREATE or INSERT OVERWRITE)
    StoreTableNode storeTableNode = PlannerUtil.findTopNode(getBlock().getPlan(), NodeType.STORE);
    if (storeTableNode != null) {
      storeType = storeTableNode.getStorageType();
    }
    meta = CatalogUtil.newTableMeta(channel.getSchema(), storeType, new Options());
    meta.setStat(stat);
    statistics = stat;
    setFinishTime();

    eventHandler.handle(new SubQuerySucceeEvent(getId(), meta));
View Full Code Here

    LogicalPlan plan = planner.createPlan(context);

    int numPartitions = 3;
    Column key1 = new Column("score.deptName", Type.TEXT);
    Column key2 = new Column("score.class", Type.TEXT);
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        PartitionType.HASH_PARTITION, numPartitions);
    dataChannel.setPartitionKey(new Column[]{key1, key2});
    ctx.setDataChannel(dataChannel);
    LogicalNode rootNode = optimizer.optimize(plan);

    TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(), StoreType.CSV);
View Full Code Here

    ctx.setEnforcer(new Enforcer());
    Expr expr = analyzer.parse(QUERIES[14]);
    LogicalPlan plan = planner.createPlan(expr);
    LogicalNode rootNode = plan.getRootBlock().getRoot();
    int numPartitions = 1;
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        PartitionType.HASH_PARTITION, numPartitions);
    dataChannel.setPartitionKey(new Column[]{});
    ctx.setDataChannel(dataChannel);
    optimizer.optimize(plan);

    TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(),
        StoreType.CSV);
View Full Code Here

    Expr context = analyzer.parse(SORT_QUERY[0]);
    LogicalPlan plan = planner.createPlan(context);
    LogicalNode rootNode = optimizer.optimize(plan);

    SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
    DataChannel channel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        PartitionType.RANGE_PARTITION);
    channel.setPartitionKey(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray());
    ctx.setDataChannel(channel);

    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
View Full Code Here

    }
  }

  private PhysicalExec buildOutputOperator(TaskAttemptContext context, LogicalNode plan,
                                           PhysicalExec execPlan) throws IOException {
    DataChannel channel = context.getDataChannel();
    ShuffleFileWriteNode shuffleFileWriteNode = LogicalPlan.createNodeWithoutPID(ShuffleFileWriteNode.class);
    shuffleFileWriteNode.setStorageType(context.getDataChannel().getStoreType());
    shuffleFileWriteNode.setInSchema(plan.getOutSchema());
    shuffleFileWriteNode.setOutSchema(plan.getOutSchema());
    shuffleFileWriteNode.setShuffle(channel.getShuffleType(), channel.getShuffleKeys(), channel.getShuffleOutputNum());
    shuffleFileWriteNode.setChild(plan);

    PhysicalExec outExecPlan = createShuffleFileWritePlan(context, shuffleFileWriteNode, execPlan);
    return outExecPlan;
  }
View Full Code Here

TOP

Related Classes of org.apache.tajo.engine.planner.global.DataChannel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.