Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.InputDescriptor

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
              new InputDescriptor(logicalInputClass.getName()));
        break;

      case CUSTOM_SIMPLE_EDGE:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;

      case SIMPLE_EDGE:
      default:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileSortedOutput.class;
        logicalInputClass = ShuffledMergedInputLegacy.class;
        break;
    }

    if (edgeProperty == null) {
      edgeProperty =
        new EdgeProperty(dataMovementType,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            new OutputDescriptor(logicalOutputClass.getName()),
            new InputDescriptor(logicalInputClass.getName()));
    }

    return edgeProperty;
  }
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
          HiveInputFormat.class.getName());
    } else {
      mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
    }
    map.addInput(alias,
        new InputDescriptor(MRInputLegacy.class.getName()).
        setUserPayload(mrInput), amSplitGeneratorClass);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(getBaseName(appJarLr), appJarLr);
    for (LocalResource lr: additionalLr) {
View Full Code Here
247
248
249
250
251
252
253
        mergeInputClass = TezMergedLogicalInput.class;
        break;
    }

    return new GroupInputEdge(group, w, createEdgeProperty(edgeProp),
        new InputDescriptor(mergeInputClass.getName()));
  }
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
              new InputDescriptor(logicalInputClass.getName()));
        break;

      case CUSTOM_SIMPLE_EDGE:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;

      case SIMPLE_EDGE:
      default:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileSortedOutput.class;
        logicalInputClass = ShuffledMergedInputLegacy.class;
        break;
    }

    if (edgeProperty == null) {
      edgeProperty =
        new EdgeProperty(dataMovementType,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            new OutputDescriptor(logicalOutputClass.getName()),
            new InputDescriptor(logicalInputClass.getName()));
    }

    return edgeProperty;
  }
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
          HiveInputFormat.class.getName());
    } else {
      mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
    }
    map.addInput(alias,
        new InputDescriptor(MRInputLegacy.class.getName()).
        setUserPayload(mrInput), amSplitGeneratorClass);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(getBaseName(appJarLr), appJarLr);
    for (LocalResource lr: additionalLr) {
View Full Code Here
246
247
248
249
250
251
252
        mergeInputClass = TezMergedLogicalInput.class;
        break;
    }

    return new GroupInputEdge(group, w, createEdgeProperty(edgeProp),
        new InputDescriptor(mergeInputClass.getName()));
  }
View Full Code Here
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
        edgeProperty =
          new EdgeProperty(edgeDesc,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
              new InputDescriptor(logicalInputClass.getName()));
        break;

      case CUSTOM_SIMPLE_EDGE:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;

      case SIMPLE_EDGE:
      default:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileSortedOutput.class;
        logicalInputClass = ShuffledMergedInputLegacy.class;
        break;
    }

    if (edgeProperty == null) {
      edgeProperty =
        new EdgeProperty(dataMovementType,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            new OutputDescriptor(logicalOutputClass.getName()),
            new InputDescriptor(logicalInputClass.getName()));
    }

    return edgeProperty;
  }
View Full Code Here
478
479
480
481
482
483
484
485
486
487
488
          HiveInputFormat.class.getName());
    } else {
      mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
    }
    map.addInput(alias,
        new InputDescriptor(MRInputLegacy.class.getName()).
        setUserPayload(mrInput), amSplitGeneratorClass);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(getBaseName(appJarLr), appJarLr);
    for (LocalResource lr: additionalLr) {
View Full Code Here
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
    if (dagVertexGroups != null && !dagVertexGroups.isEmpty()) {
      List<GroupInputSpec> groupSpecList = Lists.newLinkedList();
      for (VertexGroupInfo groupInfo : dagVertexGroups.values()) {
        if (groupInfo.edgeMergedInputs.containsKey(getName())) {
          InputDescriptor mergedInput = groupInfo.edgeMergedInputs.get(getName());
          groupSpecList.add(new GroupInputSpec(groupInfo.groupName,
              Lists.newLinkedList(groupInfo.groupMembers), mergedInput));
        }
      }
      if (!groupSpecList.isEmpty()) {
View Full Code Here
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
    Preconditions.checkArgument(inputs.size() < 2,
        "For now, only a single root input can be specified on a Vertex");
    this.additionalInputs = Maps.newHashMapWithExpectedSize(inputs.size());
    for (RootInputLeafOutputProto input : inputs) {

      InputDescriptor id = DagTypeConverters
          .convertInputDescriptorFromDAGPlan(input.getEntityDescriptor());

      this.additionalInputs.put(input.getName(),
          new RootInputLeafOutputDescriptor<InputDescriptor>(input.getName(), id,
              input.hasInitializerClassName() ? input.getInitializerClassName()
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.InputDescriptor

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.tez.dag.app.dag.impl.TestDAGImpl
  • org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager
  • org.apache.tez.dag.app.dag.impl.TestVertexScheduler
  • org.apache.tez.dag.app.dag.impl.VertexImpl
  • org.apache.tez.dag.app.dag.impl.VertexImpl$InitTransition
  • org.apache.tez.dag.app.rm.TestContainerReuse
  • org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.