Package org.apache.tez.runtime.library.output

Examples of org.apache.tez.runtime.library.output.UnorderedKVOutput


      throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput");
    }

    MRInputLegacy mrInput = (MRInputLegacy) li;
    mrInput.init();
    UnorderedKVOutput kvOutput = (UnorderedKVOutput) lo;

    Configuration updatedConf = mrInput.getConfigUpdates();
    Text srcFile = new Text();
    srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
    if (updatedConf != null) {
      String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
      if (fileName != null) {
        LOG.info("Processing file: " + fileName);
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();
View Full Code Here


    }

    @Override
    public void run() throws Exception {
      Preconditions.checkArgument(getOutputs().size() == 1);
      UnorderedKVOutput output = (UnorderedKVOutput) getOutputs().values().iterator()
          .next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
      ByteBuffer userPayload = getContext().getUserPayload().getPayload();
      if (userPayload != null) {
        boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false;
        if (doLocalityCheck) {
View Full Code Here

TOP

Related Classes of org.apache.tez.runtime.library.output.UnorderedKVOutput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.