Package org.apache.tez.runtime.library.api

Examples of org.apache.tez.runtime.library.api.KeyValueWriter

270
271
272
273
274
275
276
277
278
279
280
        "-" + taskNumberFormat.format(outputContext.getTaskIndex());
  }

  @Override
  public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {
      private final boolean useNewWriter = useNewApi;

      @SuppressWarnings("unchecked")
      @Override
      public void write(Object key, Object value) throws IOException {
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
      for (LogicalOutput output : outputs.values()) {
        output.start();
      }

      MROutput out = (MROutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = out.getWriter();
      KeyValuesReader kvReader = (KeyValuesReader) inputs.values().iterator().next().getReader();
      while (kvReader.next()) {
        Text word = (Text) kvReader.getCurrentKey();
        int sum = 0;
        for (Object value : kvReader.getCurrentValues()) {
          sum += ((IntWritable) value).get();
        }
        kvWriter.write(word, new IntWritable(sum));
      }
      if (out.isCommitRequired()) {
        while (!context.canCommit()) {
          Thread.sleep(100);
        }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }
    ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
    KeyValuesReader kvReader = shuffleInput.getReader();

    KeyValueWriter kvWriter = null;
    if((out instanceof MROutput)) {
      kvWriter = ((MROutput) out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)) {
      kvWriter = ((OnFileSortedOutput) out).getWriter();
    } else {
View Full Code Here
251
252
253
254
255
256
257
258
259
260
261
    return "part-" + NUMBER_FORMAT.format(outputContext.getTaskIndex());
  }

  @Override
  public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {
      private final boolean useNewWriter = useNewApi;

      @SuppressWarnings("unchecked")
      @Override
      public void write(Object key, Object value) throws IOException {
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
    return Collections.emptyList();
  }

  @Override
  public Writer getWriter() throws IOException {
    return new KeyValueWriter() {
     
      @Override
      public void write(Object key, Object value) throws IOException {
        sorter.write(key, value);
      }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    return Collections.emptyList();
  }

  @Override
  public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {
      @Override
      public void write(Object key, Object value) throws IOException {
        sorter.write(key, value);
      }
    };
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();

      Text valText = (Text) val;
      String readVal = valText.toString();
      if (readVal.contains(filterWord)) {
        LongWritable lineNum = (LongWritable) key;
        TextLongPair outVal = new TextLongPair(srcFile, lineNum);
        kvWriter.write(valText, outVal);
      }
    }
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;
    MROutput mrOutput = (MROutput) lo;

    KeyValueReader kvReader = kvInput.getReader();
    KeyValueWriter kvWriter = mrOutput.getWriter();
    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object value = kvReader.getCurrentValue();

      kvWriter.write(key, value);
    }
    if (processorContext.canCommit()) {
      mrOutput.commit();
    } else {
      mrOutput.abort();
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.api.KeyValueWriter

  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez
  • org.apache.tez.examples.JoinDataGen$GenDataProcessor
  • org.apache.tez.examples.JoinExample$ForwardingProcessor
  • org.apache.tez.examples.JoinExample$JoinProcessor
  • org.apache.tez.examples.OrderedWordCount$NoOpSorter
  • org.apache.tez.examples.OrderedWordCount$SumProcessor
  • org.apache.tez.examples.WordCount$SumProcessor
  • org.apache.tez.examples.WordCount$TokenProcessor
  • org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample$InputProcessor
  • org.apache.tez.mapreduce.examples.processor.FilterByWordInputProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.