Examples of org.apache.tez.runtime.api.LogicalInput

org.apache.tez.runtime.api.LogicalInput
An {@link Input} which handles all incoming physical connections on anedge. A {@link LogicalIOProcessor} sees a single Logical Input per incomingedge. It provides a unified view that hides the details of the actual physical inputs User implementations are expected to derive from {@link AbstractLogicalInput}

      MROutput out = (MROutput) outputs.get("union");
      MROutput allParts = (MROutput) outputs.get("all-parts");
      KeyValueWriter kvWriter = out.getWriter();
      KeyValueWriter partsWriter = allParts.getWriter();
      Map<String, AtomicInteger> unionKv = Maps.newHashMap();
      LogicalInput union = inputs.get("union");
      KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
      while (kvReader.next()) {
        String word = ((Text) kvReader.getCurrentKey()).toString();
        IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
        for (int i=0; i<intVal.get(); ++i) {
          partsWriter.write(word, one);
        }
        AtomicInteger value = unionKv.get(word);
        if (value == null) {
          unionKv.put(word, new AtomicInteger(intVal.get()));
        } else {
          value.addAndGet(intVal.get());
        }
      }
      LogicalInput map3 = inputs.get("map3");
      kvReader = (KeyValuesReader) map3.getReader();
      while (kvReader.next()) {
        String word = ((Text) kvReader.getCurrentKey()).toString();
        IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
        AtomicInteger value = unionKv.get(word);
        if  (value == null) {

View Full Code Here

    }
    for (LogicalOutput output : outputs.values()) {
      output.start();
    }


    LogicalInput li = inputs.values().iterator().next();
    if (! (li instanceof ShuffledUnorderedKVInput)) {
      throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput");
    }


    LogicalOutput lo = outputs.values().iterator().next();

View Full Code Here

      if (pos == desc.getPosBigTable()) {
        continue;
      }


      String inputName = parentToInput.get(pos);
      LogicalInput input = tezContext.getInput(inputName);


      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();


        MapJoinTableContainer tableContainer = new HashMapWrapper(hashTableThreshold,
            hashTableLoadFactor);


        // simply read all the kv pairs into the hashtable.

View Full Code Here


    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf)
            throws ExecException {
        LogicalInput input = inputs.get(inputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + inputKey + " is missing");
        }
        try {
            reader = (KeyValuesReader) input.getReader();
            LOG.info("Attached input from vertex " + inputKey + " : input=" + input + ", reader=" + reader);
        } catch (Exception e) {
            throw new ExecException(e);
        }
    }

View Full Code Here

            Configuration conf) throws ExecException {
        this.conf = conf;
        List<KeyValueReader> readersList = new ArrayList<KeyValueReader>();
        try {
            for (String inputKey : inputKeys) {
                LogicalInput input = inputs.get(inputKey);
                if (input == null) {
                    throw new ExecException("Input from vertex " + inputKey
                            + " is missing");
                }


                KeyValueReader reader = (KeyValueReader) input.getReader();
                readersList.add(reader);
                LOG.info("Attached input from vertex " + inputKey + " : input="
                        + input + ", reader=" + reader);
            }
            readers = readersList.iterator();

View Full Code Here

    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf)
            throws ExecException {
        this.conf = conf;
        LogicalInput logInput = inputs.get(inputKey);
        if (logInput == null || !(logInput instanceof MRInput)) {
            throw new ExecException("POSimpleTezLoad only accepts MRInputs");
        }
        input = (MRInput) logInput;
        try {

View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf) throws ExecException {
        this.conf = conf;
        LogicalInput input = inputs.get(tuplesInputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + tuplesInputKey + " is missing");
        }
        try {
            reader = (KeyValueReader) input.getReader();
            LOG.info("Attached input from vertex " + tuplesInputKey + " : input=" + input + ", reader=" + reader);
        } catch (Exception e) {
            throw new ExecException(e);
        }


        String cacheKey = "rankstats-" + getOperatorKey().toString();
        if (isInputCached) {
            counterOffsets = (Map<Integer, Long>) ObjectCache.getInstance().retrieve(cacheKey);
            LOG.info("Found counter stats for PORankTez in Tez cache. cachekey=" + cacheKey);
            return;
        }
        input = inputs.get(statsInputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + statsInputKey + " is missing");
        }
        try {
            KeyValueReader reader = (KeyValueReader) input.getReader();
            LOG.info("Attached input from vertex " + statsInputKey + " : input=" + input + ", reader=" + reader);
            reader.next();
            // POCounterStatsTez produces a HashMap which contains
            // mapping of task id and the offset of record count in each task based on total record count
            Map<String, Long> counterOffsetsTemp = (Map<String, Long>) ((Tuple)reader.getCurrentValue()).get(0);

View Full Code Here

            throws ExecException {
        this.conf = conf;
        comparator = (WritableComparator) ConfigUtils.getInputKeySecondaryGroupingComparator(conf);
        try {
            for (String key : inputKeys) {
                LogicalInput input = inputs.get(key);
                this.inputs.add(input);
                this.readers.add((KeyValuesReader)input.getReader());
            }


            // We need to adjust numInputs because it's possible for both
            // OrderedGroupedKVInput and non-OrderedGroupedKVInput to be attached
            // to the same vertex. If so, we're only interested in

View Full Code Here

    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf)
            throws ExecException {
        this.conf = conf;
        LogicalInput input = inputs.get(inputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + inputKey + " is missing");
        }


        try {
            Reader r = input.getReader();
            if (r instanceof KeyValueReader) {
                reader = (KeyValueReader) r;
            } else {
                shuffleInput = true;
                shuffleReader = (KeyValuesReader) r;

View Full Code Here

    }


    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf) throws ExecException {
        LogicalInput input = inputs.get(inputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + inputKey + " is missing");
        }
        try {
            Reader r = input.getReader();
            if (r instanceof KeyValueReader) {
                reader = (KeyValueReader) r;
            } else {
                shuffleInput = true;
                shuffleReader = (KeyValuesReader) r;

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.tez.runtime.api.LogicalInput

cascading.flow.tez.stream.element.TezGroupByGate

cascading.flow.tez.stream.element.TezMergeGate

cascading.flow.tez.stream.graph.Hadoop2TezStreamGraph

org.apache.hadoop.hive.ql.exec.tez.HashTableLoader

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POCounterStatsTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POFRJoinTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POIdentityInOutTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.PORankTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffledValueInputTez

org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffleTezLoad

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.