Package org.apache.tez.runtime.library.api

Examples of org.apache.tez.runtime.library.api.KeyValueWriter

131
132
133
134
135
136
137
138
139
140
141
142
143
144
    List<Event> events = null;

    events = kvOutput.initialize();
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
      Preconditions.checkArgument(getOutputs().size() == (inUnion ? 2 : 1));
      Preconditions.checkArgument(getOutputs().containsKey("checker"));
      MRInput input = (MRInput) getInputs().values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      Output output =  getOutputs().get("checker");
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      MROutput parts = null;
      KeyValueWriter partsWriter = null;
      if (inUnion) {
        parts = (MROutput) getOutputs().get("parts");
        partsWriter = parts.getWriter();
      }
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
          if (inUnion) {
            partsWriter.write(word, one);
          }
        }
      }
    }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    public void run() throws Exception {
      Preconditions.checkArgument(getInputs().size() == 2);
      Preconditions.checkArgument(getOutputs().size() == 2);
      MROutput out = (MROutput) getOutputs().get("union");
      MROutput allParts = (MROutput) getOutputs().get("all-parts");
      KeyValueWriter kvWriter = out.getWriter();
      KeyValueWriter partsWriter = allParts.getWriter();
      Map<String, AtomicInteger> unionKv = Maps.newHashMap();
      LogicalInput union = getInputs().get("union");
      KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
      while (kvReader.next()) {
        String word = ((Text) kvReader.getCurrentKey()).toString();
        IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
        for (int i = 0; i < intVal.get(); ++i) {
          partsWriter.write(word, one);
        }
        AtomicInteger value = unionKv.get(word);
        if (value == null) {
          unionKv.put(word, new AtomicInteger(intVal.get()));
        } else {
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();

      Text valText = (Text) val;
      String readVal = valText.toString();
      if (readVal.contains(filterWord)) {
        LongWritable lineNum = (LongWritable) key;
        TextLongPair outVal = new TextLongPair(srcFile, lineNum);
        kvWriter.write(valText, outVal);
      }
    }
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
    UnorderedKVInput kvInput = (UnorderedKVInput) li;
    MROutput mrOutput = (MROutput) lo;

    KeyValueReader kvReader = kvInput.getReader();
    KeyValueWriter kvWriter = mrOutput.getWriter();
    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object value = kvReader.getCurrentValue();

      kvWriter.write(key, value);
    }
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
    @Override
    public void run() throws Exception {
      Preconditions.checkArgument(getOutputs().size() == 1);
      UnorderedKVOutput output = (UnorderedKVOutput) getOutputs().values().iterator()
          .next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
      ByteBuffer userPayload = getContext().getUserPayload().getPayload();
      if (userPayload != null) {
        boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false;
        if (doLocalityCheck) {
          ObjectRegistry objectRegistry = getContext().getObjectRegistry();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    }
  }

  @Override
  public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {
      @Override
      public void write(Object key, Object value) throws IOException {
        sorter.write(key, value);
      }
    };
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
      Preconditions.checkArgument(inputs.size() == 1);
      Preconditions.checkArgument(outputs.size() == 1);
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
        }
      }
    }
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }
    ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
    KeyValuesReader kvReader = shuffleInput.getReader();

    KeyValueWriter kvWriter = null;
    if((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy) out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)) {
      kvWriter = ((OnFileSortedOutput) out).getWriter();
    } else {
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }

    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy)out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)){
      kvWriter = ((OnFileSortedOutput)out).getWriter();
    } else {
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.api.KeyValueWriter

  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez
  • org.apache.tez.examples.JoinDataGen$GenDataProcessor
  • org.apache.tez.examples.JoinExample$ForwardingProcessor
  • org.apache.tez.examples.JoinExample$JoinProcessor
  • org.apache.tez.examples.OrderedWordCount$NoOpSorter
  • org.apache.tez.examples.OrderedWordCount$SumProcessor
  • org.apache.tez.examples.WordCount$SumProcessor
  • org.apache.tez.examples.WordCount$TokenProcessor
  • org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample$InputProcessor
  • org.apache.tez.mapreduce.examples.processor.FilterByWordInputProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.