Package org.apache.tez.runtime.library.api

Examples of org.apache.tez.runtime.library.api.KeyValueReader

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        output.start();
      }
      Preconditions.checkArgument(inputs.size() == 1);
      Preconditions.checkArgument(outputs.size() == 1);
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
        }
      }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
  }

  @Override
  public KeyValueReader getReader() throws Exception {
    if (numInputs == 0) {
      return new KeyValueReader() {
        @Override
        public boolean next() throws IOException {
          return false;
        }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
  }

  @Override
  public KeyValueReader getReader() throws Exception {
    if (numInputs == 0) {
      return new KeyValueReader() {
        @Override
        public boolean next() throws IOException {
          return false;
        }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
        LOG.info("Processing file: " + fileName);
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();

      Text valText = (Text) val;
      String readVal = valText.toString();
      if (readVal.contains(filterWord)) {
        LongWritable lineNum = (LongWritable) key;
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    }

    ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;
    MROutput mrOutput = (MROutput) lo;

    KeyValueReader kvReader = kvInput.getReader();
    KeyValueWriter kvWriter = mrOutput.getWriter();
    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object value = kvReader.getCurrentValue();

      kvWriter.write(key, value);
    }
    if (processorContext.canCommit()) {
      mrOutput.commit();
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
        inUnion = false;
      }
      Preconditions.checkArgument(outputs.size() == (inUnion ? 2 : 1));
      Preconditions.checkArgument(outputs.containsKey("checker"));
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.get("checker");
      KeyValueWriter kvWriter = output.getWriter();
      MROutput parts = null;
      KeyValueWriter partsWriter = null;
      if (inUnion) {
        parts = (MROutput) outputs.get("parts");
        partsWriter = parts.getWriter();
      }
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
          if (inUnion) {
            partsWriter.write(word, one);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
        output.start();
      }
      Preconditions.checkArgument(inputs.size() == 1);
      Preconditions.checkArgument(outputs.size() == 1);
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
        }
      }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
        LOG.info("Processing file: " + fileName);
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();

      Text valText = (Text) val;
      String readVal = valText.toString();
      if (readVal.contains(filterWord)) {
        LongWritable lineNum = (LongWritable) key;
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    }

    ShuffledUnorderedKVInput kvInput = (ShuffledUnorderedKVInput) li;
    MROutput mrOutput = (MROutput) lo;

    KeyValueReader kvReader = kvInput.getReader();
    KeyValueWriter kvWriter = mrOutput.getWriter();
    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object value = kvReader.getCurrentValue();

      kvWriter.write(key, value);
    }
    if (processorContext.canCommit()) {
      mrOutput.commit();
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
  @Override
  public synchronized KeyValueReader getReader() throws Exception {
    Preconditions.checkState(isStarted.get(), "Must start input before invoking this method");
    if (numInputs == 0) {
      return new KeyValueReader() {
        @Override
        public boolean next() throws IOException {
          return false;
        }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.api.KeyValueReader

  • cascading.flow.tez.stream.element.TezBoundaryStage
  • cascading.flow.tez.stream.element.TezMergeGate
  • org.apache.hadoop.hive.ql.exec.tez.HashTableLoader
  • org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor
  • org.apache.hadoop.hive.ql.exec.tez.MergeFileRecordProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.PORankTez
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffledValueInputTez
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez
  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor
  • org.apache.tez.examples.JoinExample$ForwardingProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.