Package org.apache.tajo.engine.planner.global

Examples of org.apache.tajo.engine.planner.global.DataChannel

371
372
373
374
375
376
377
378
379
380
381
    private void finalizeQuery(Query query, QueryCompletedEvent event) {
      MasterPlan masterPlan = query.getPlan();

      ExecutionBlock terminal = query.getPlan().getTerminalBlock();
      DataChannel finalChannel = masterPlan.getChannel(event.getExecutionBlockId(), terminal.getId());
      Path finalOutputDir = commitOutputData(query);

      QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext());
      try {
        hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(),
View Full Code Here
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
      statsArray = computeStatFromUnionBlock(this);
    } else {
      statsArray = computeStatFromTasks();
    }

    DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0);
    // get default or store type
    CatalogProtos.StoreType storeType = CatalogProtos.StoreType.CSV; // default setting

    // if store plan (i.e., CREATE or INSERT OVERWRITE)
    StoreTableNode storeTableNode = PlannerUtil.findTopNode(getBlock().getPlan(), NodeType.STORE);
    if (storeTableNode != null) {
      storeType = storeTableNode.getStorageType();
    }
    schema = channel.getSchema();
    meta = CatalogUtil.newTableMeta(storeType, new Options());
    inputStatistics = statsArray[0];
    resultStatistics = statsArray[1];
  }
View Full Code Here
628
629
630
631
632
633
634
635
636
637
638
        if (execBlock.hasUnion()) {
          subQuery.finalizeStats();
          state = SubQueryState.SUCCEEDED;
        } else {
          ExecutionBlock parent = subQuery.getMasterPlan().getParent(subQuery.getBlock());
          DataChannel channel = subQuery.getMasterPlan().getChannel(subQuery.getId(), parent.getId());
          setShuffleIfNecessary(subQuery, channel);
          initTaskScheduler(subQuery);
          schedule(subQuery);
          subQuery.totalScheduledObjectsCount = subQuery.getTaskScheduler().remainingScheduledObjectNum();
          LOG.info(subQuery.totalScheduledObjectsCount + " objects are scheduled");
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
334
335
336
  }

  public static void scheduleFragmentsForNonLeafTasks(TaskSchedulerContext schedulerContext,
                                                      MasterPlan masterPlan, SubQuery subQuery, int maxNum)
      throws IOException {
    DataChannel channel = masterPlan.getIncomingChannels(subQuery.getBlock().getId()).get(0);
    if (channel.getShuffleType() == HASH_SHUFFLE) {
      scheduleHashShuffledFetches(schedulerContext, masterPlan, subQuery, channel, maxNum);
    } else if (channel.getShuffleType() == RANGE_SHUFFLE) {
      scheduleRangeShuffledFetches(schedulerContext, masterPlan, subQuery, channel, maxNum);
    } else {
      throw new InternalException("Cannot support partition type");
    }
  }
View Full Code Here
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
    LogicalPlan plan = planner.createPlan(session, context);

    int numPartitions = 3;
    Column key1 = new Column("default.score.deptname", Type.TEXT);
    Column key2 = new Column("default.score.class", Type.TEXT);
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        ShuffleType.HASH_SHUFFLE, numPartitions);
    dataChannel.setShuffleKeys(new Column[]{key1, key2});
    ctx.setDataChannel(dataChannel);
    LogicalNode rootNode = optimizer.optimize(plan);

    TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());

    FileSystem fs = sm.getFileSystem();

    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
View Full Code Here
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
    ctx.setEnforcer(new Enforcer());
    Expr expr = analyzer.parse(QUERIES[14]);
    LogicalPlan plan = planner.createPlan(session, expr);
    LogicalNode rootNode = plan.getRootBlock().getRoot();
    int numPartitions = 1;
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        ShuffleType.HASH_SHUFFLE, numPartitions);
    dataChannel.setShuffleKeys(new Column[]{});
    ctx.setDataChannel(dataChannel);
    optimizer.optimize(plan);

    TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());

    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
    exec.init();
    exec.next();
View Full Code Here
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
    Expr context = analyzer.parse(SORT_QUERY[0]);
    LogicalPlan plan = planner.createPlan(session, context);
    LogicalNode rootNode = optimizer.optimize(plan);

    SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
    DataChannel channel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE);
    channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray());
    ctx.setDataChannel(channel);

    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);

    Tuple tuple;
    exec.init();
    exec.next();
    exec.close();

    Schema keySchema = new Schema();
    keySchema.addColumn("?empId", Type.INT4);
    SortSpec[] sortSpec = new SortSpec[1];
    sortSpec[0] = new SortSpec(keySchema.getColumn(0), true, false);
    TupleComparator comp = new TupleComparator(keySchema, sortSpec);
    BSTIndex bst = new BSTIndex(conf);
    BSTIndex.BSTIndexReader reader = bst.getIndexReader(new Path(workDir, "output/index"),
        keySchema, comp);
    reader.open();
    Path outputPath = StorageUtil.concatPath(workDir, "output", "output");
    TableMeta meta = CatalogUtil.newTableMeta(channel.getStoreType(), new Options());
    SeekableScanner scanner =
        StorageManagerFactory.getSeekableScanner(conf, meta, exec.getSchema(), outputPath);
    scanner.init();

    int cnt = 0;
View Full Code Here
217
218
219
220
221
222
223
224
      return dataChannel;
    }
    if (!p.hasDataChannel()) {
      return null;
    }
    this.dataChannel = new DataChannel(p.getDataChannel());
    return this.dataChannel;
  }
View Full Code Here
TOP

Related Classes of org.apache.tajo.engine.planner.global.DataChannel

  • org.apache.tajo.catalog.Column
  • org.apache.tajo.catalog.Schema
  • org.apache.tajo.engine.planner.physical.TestPhysicalPlanner
  • org.apache.tajo.engine.planner.PhysicalPlannerImpl
  • org.apache.tajo.engine.query.QueryUnitRequestImpl
  • org.apache.tajo.ExecutionBlockId
  • org.apache.tajo.master.querymaster.Query$QueryCompletedTransition
  • org.apache.tajo.master.querymaster.Query$SubQueryCompletedTransition
  • org.apache.tajo.master.querymaster.Repartitioner
  • org.apache.tajo.master.querymaster.SubQuery
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.