Package org.apache.tez.runtime.library.api

Examples of org.apache.tez.runtime.library.api.KeyValueReader

89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
      String inputName = parentToInput.get(pos);
      LogicalInput input = tezContext.getInput(inputName);

      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();

        MapJoinTableContainer tableContainer = new HashMapWrapper(hashTableThreshold,
            hashTableLoadFactor);

        // simply read all the kv pairs into the hashtable.

        while (kvReader.next()) {
          // We pass key in as reference, to find out quickly if optimized keys can be used.
          // However, we do not reuse the object since we are putting them into the hashmap.
          lastKey = MapJoinKey.read(output, lastKey, mapJoinTableSerdes[pos].getKeyContext(),
              (Writable)kvReader.getCurrentKey(), false);

          LazyFlatRowContainer values = (LazyFlatRowContainer)tableContainer.get(lastKey);
          if (values == null) {
            values = new LazyFlatRowContainer();
            tableContainer.put(lastKey, values);
          }
          values.add(mapJoinTableSerdes[pos].getValueContext(),
              (BytesWritable)kvReader.getCurrentValue(), useLazyRows);
        }

        mapJoinTables[pos] = tableContainer;
      } catch (IOException e) {
        throw new HiveException(e);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
  @Override
  void run() throws IOException{

    MRInputLegacy in = TezProcessor.getMRInput(inputs);
    KeyValueReader reader = in.getReader();

    //process records until done
    while(reader.next()){
      //ignore the key for maps -  reader.getCurrentKey();
      Object value = reader.getCurrentValue();
      boolean needMore = processRow(value);
      if(!needMore){
        break;
      }
    }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
                if (input == null) {
                    throw new ExecException("Input from vertex " + inputKey
                            + " is missing");
                }

                KeyValueReader reader = (KeyValueReader) input.getReader();
                readersList.add(reader);
                LOG.info("Attached input from vertex " + inputKey + " : input="
                        + input + ", reader=" + reader);
            }
            readers = readersList.iterator();
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        input = inputs.get(statsInputKey);
        if (input == null) {
            throw new ExecException("Input from vertex " + statsInputKey + " is missing");
        }
        try {
            KeyValueReader reader = (KeyValueReader) input.getReader();
            LOG.info("Attached input from vertex " + statsInputKey + " : input=" + input + ", reader=" + reader);
            reader.next();
            // POCounterStatsTez produces a HashMap which contains
            // mapping of task id and the offset of record count in each task based on total record count
            Map<String, Long> counterOffsetsTemp = (Map<String, Long>) ((Tuple)reader.getCurrentValue()).get(0);
            counterOffsets = new HashMap<Integer, Long>(counterOffsetsTemp.size(), 1);
            for (Entry<String, Long> entry : counterOffsetsTemp.entrySet()) {
                counterOffsets.put(Integer.valueOf(entry.getKey()), entry.getValue());
            }
            ObjectCache.getInstance().cache(cacheKey, counterOffsets);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
      String inputName = parentToInput.get(pos);
      LogicalInput input = tezContext.getInput(inputName);

      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();
        MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(),
          valCtx = mapJoinTableSerdes[pos].getValueContext();
        if (useOptimizedTables) {
          ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector();
          if (!MapJoinBytesTableContainer.isSupportedKey(keyOi)) {
            if (isFirstKey) {
              useOptimizedTables = false;
            } else {
              throw new HiveException(describeOi(
                  "Only a subset of mapjoin keys is supported. Unsupported key: ", keyOi));
            }
          }
        }
        isFirstKey = false;
        Long keyCountObj = parentKeyCounts.get(pos);
        long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
        MapJoinTableContainer tableContainer = useOptimizedTables
            ? new MapJoinBytesTableContainer(hconf, valCtx, keyCount, memUsage)
            : new HashMapWrapper(hconf, keyCount);

        while (kvReader.next()) {
          rowCount++;
          lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
              valCtx, (Writable)kvReader.getCurrentValue());
        }

        tableContainer.seal();
        mapJoinTables[pos] = tableContainer;
      } catch (IOException e) {
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
    int size = mergeMapOpList.size() + 1; // the +1 is for the main map operator itself
    sources = new MapRecordSource[size];
    position = mapOp.getConf().getTag();
    sources[position] = new MapRecordSource();
    KeyValueReader reader = null;
    if (mainWorkMultiMRInput != null) {
      reader = getKeyValueReader(mainWorkMultiMRInput.getKeyValueReaders(), mapOp);
    } else {
      reader = legacyMRInput.getReader();
    }
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
      throws Exception {
    List<KeyValueReader> kvReaderList = new ArrayList<KeyValueReader>(keyValueReaders);
    // this sets up the map operator contexts correctly
    mapOp.initializeContexts();
    Deserializer deserializer = mapOp.getCurrentDeserializer();
    KeyValueReader reader =
        new KeyValueInputMerger(kvReaderList, deserializer,
            new ObjectInspector[] { deserializer.getObjectInspector() }, mapOp
                .getConf()
                .getSortCols());
    return reader;
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
    perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS);
  }

  @Override
  void run() throws Exception {
    KeyValueReader reader = mrInput.getReader();

    //process records until done
    while (reader.next()) {
      boolean needMore = processRow(reader.getCurrentKey(),
          reader.getCurrentValue());
      if (!needMore) {
        break;
      }
    }
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
      String inputName = parentToInput.get(pos);
      LogicalInput input = tezContext.getInput(inputName);

      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();

        MapJoinTableContainer tableContainer = new HashMapWrapper(hashTableThreshold,
            hashTableLoadFactor);

        // simply read all the kv pairs into the hashtable.

        while (kvReader.next()) {
          // We pass key in as reference, to find out quickly if optimized keys can be used.
          // However, we do not reuse the object since we are putting them into the hashmap.
          lastKey = MapJoinKey.read(output, lastKey, mapJoinTableSerdes[pos].getKeyContext(),
              (Writable)kvReader.getCurrentKey(), false);

          LazyFlatRowContainer values = (LazyFlatRowContainer)tableContainer.get(lastKey);
          if (values == null) {
            values = new LazyFlatRowContainer();
            tableContainer.put(lastKey, values);
          }
          values.add(mapJoinTableSerdes[pos].getValueContext(),
              (BytesWritable)kvReader.getCurrentValue(), useLazyRows);
        }

        mapJoinTables[pos] = tableContainer;
      } catch (IOException e) {
        throw new HiveException(e);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
  @Override
  void run() throws IOException{

    MRInputLegacy in = TezProcessor.getMRInput(inputs);
    KeyValueReader reader = in.getReader();

    //process records until done
    while(reader.next()){
      //ignore the key for maps -  reader.getCurrentKey();
      Object value = reader.getCurrentValue();
      boolean needMore = processRow(value);
      if(!needMore){
        break;
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.api.KeyValueReader

  • cascading.flow.tez.stream.element.TezBoundaryStage
  • cascading.flow.tez.stream.element.TezMergeGate
  • org.apache.hadoop.hive.ql.exec.tez.HashTableLoader
  • org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor
  • org.apache.hadoop.hive.ql.exec.tez.MergeFileRecordProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.PORankTez
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POShuffledValueInputTez
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez
  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor
  • org.apache.tez.examples.JoinExample$ForwardingProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.