Examples of org.apache.tez.runtime.api.LogicalInput

org.apache.tez.runtime.api.LogicalInput
An {@link Input} which handles all incoming physical connections on anedge. A {@link LogicalIOProcessor} sees a single Logical Input per incomingedge. It provides a unified view that hides the details of the actual physical inputs User implementations are expected to derive from {@link AbstractLogicalInput}

    


    // then add the non-grouped inputs
    for (InputSpec inputSpec : inputSpecs) {
      if (!groupInputs.contains(inputSpec.getSourceVertexName())) {
        LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName());
        runInputMap.put(inputSpec.getSourceVertexName(), input);
      }
    }


    for (OutputSpec outputSpec : outputSpecs) {

View Full Code Here

          + ", eventDestinationInfo=" + e.getDestinationInfo());
    }
    try {
      switch (e.getDestinationInfo().getEventGenerator()) {
      case INPUT:
        LogicalInput input = inputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (input != null) {
          input.handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;

View Full Code Here

        || outputs.size() != 1) {
      throw new IOException("Cannot handle multiple inputs or outputs"
          + ", inputCount=" + inputs.size()
          + ", outputCount=" + outputs.size());
    }
    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();


    initTask(out);


    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
      throw new IOException(new TezException(
          "Only MRInputLegacy supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy)in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }


    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy)out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)){
      kvWriter = ((OnFileSortedOutput)out).getWriter();
    } else {
      throw new IOException("Illegal output to map: " + in.getClass());
    }


    if (useNewApi) {
      runNewMapper(jobConf, mrReporter, input, kvWriter);
    } else {

View Full Code Here

    if (inputs.size() <= 0 || inputs.size() > 1) {
      throw new IOException("Invalid number of inputs"
          + ", inputCount=" + inputs.size());
    }


    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();


    this.statusUpdate();


    Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
    Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
    LOG.info("Using keyClass: " + keyClass);
    LOG.info("Using valueClass: " + valueClass);
    RawComparator comparator =
        ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
    LOG.info("Using comparator: " + comparator);


    reduceInputKeyCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
    reduceInputValueCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);


    // Sanity check
    if (!(in instanceof ShuffledMergedInputLegacy)) {
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }
    ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
    KeyValuesReader kvReader = shuffleInput.getReader();


    KeyValueWriter kvWriter = null;
    if((out instanceof MROutput)) {
      kvWriter = ((MROutput) out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)) {
      kvWriter = ((OnFileSortedOutput) out).getWriter();
    } else {
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }


    if (useNewApi) {
      try {
        runNewReducer(

View Full Code Here


    if (outputs.size() != 1) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single output");
    }


    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof MRInput)) {
      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();

View Full Code Here


    if (outputs.size() != 1) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single output");
    }


    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof ShuffledUnorderedKVInput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();

View Full Code Here

    }
    LOG.info("All initializers finished");


    // Construct Inputs/Outputs map argument for processor.run()
    for (InputSpec inputSpec : inputSpecs) {
      LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName());
      runInputMap.put(inputSpec.getSourceVertexName(), input);
    }
    for (OutputSpec outputSpec : outputSpecs) {
      LogicalOutput output = outputsMap.get(outputSpec.getDestinationVertexName());
      runOutputMap.put(outputSpec.getDestinationVertexName(), output);

View Full Code Here

          + ", eventDestinationInfo=" + e.getDestinationInfo());
    }
    try {
      switch (e.getDestinationInfo().getEventGenerator()) {
      case INPUT:
        LogicalInput input = inputsMap.get(
            e.getDestinationInfo().getEdgeVertexName());
        if (input != null) {
          input.handleEvents(Collections.singletonList(e.getEvent()));
        } else {
          throw new TezUncheckedException("Unhandled event for invalid target: "
              + e);
        }
        break;

View Full Code Here


    @Override
    public Void call() throws Exception {
      LOG.info("Initializing Input using InputSpec: " + inputSpec);
      String edgeName = inputSpec.getSourceVertexName();
      LogicalInput input = createInput(inputSpec);
      TezInputContext inputContext = createInputContext(inputSpec);
      inputsMap.put(edgeName, input);
      inputContextMap.put(edgeName, inputContext);


      if (input instanceof LogicalInput) {
        ((LogicalInput) input).setNumPhysicalInputs(inputSpec
            .getPhysicalEdgeCount());
      }
      LOG.info("Initializing Input with src edge: " + edgeName);
      List<Event> events = input.initialize(inputContext);
      sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT,
          inputContext.getTaskVertexName(), inputContext.getSourceVertexName(),
          taskSpec.getTaskAttemptID());
      LOG.info("Initialized Input with src edge: " + edgeName);
      return null;

View Full Code Here

        || outputs.size() != 1) {
      throw new IOException("Cannot handle multiple inputs or outputs"
          + ", inputCount=" + inputs.size()
          + ", outputCount=" + outputs.size());
    }
    LogicalInput in = inputs.values().iterator().next();
    LogicalOutput out = outputs.values().iterator().next();


    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
      throw new IOException(new TezException(
          "Only Simple Input supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy)in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.tez.runtime.api.LogicalInput

cascading.flow.tez.stream.element.TezGroupByGate

cascading.flow.tez.stream.element.TezMergeGate

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

org.apache.hadoop.hive.ql.exec.tez.HashTableLoader

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterStatsTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POFRJoinTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POIdentityInOutTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.PORankTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffledValueInputTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffleTezLoad

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.