Package org.apache.tez.runtime.library.common.shuffle.impl

Examples of org.apache.tez.runtime.library.common.shuffle.impl.InMemoryReader

480
481
482
483
484
485
486
487
488
    readUsingInMemoryReader(bytes, originalData);
  }

  private void readUsingInMemoryReader(byte[] bytes, List<KVPair> originalData)
      throws IOException {
    InMemoryReader inMemReader = new InMemoryReader(null,
        new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    verifyData(inMemReader, originalData);
  }
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
  public IFile.Reader openIFileReader(FetchedInput fetchedInput)
      throws IOException {
    if (fetchedInput.getType() == Type.MEMORY) {
      MemoryFetchedInput mfi = (MemoryFetchedInput) fetchedInput;

      return new InMemoryReader(null, mfi.getInputAttemptIdentifier(),
          mfi.getBytes(), 0, (int) mfi.getActualSize());
    } else {
      return new IFile.Reader(fetchedInput.getInputStream(),
          fetchedInput.getCompressedSize(), codec, null, null, ifileReadAhead,
          ifileReadAheadLength, ifileBufferSize);
View Full Code Here
362
363
364
365
366
367
368
369
370
371
372
        keyBuf.reset();
        valBuf.reset();
        keyIn.reset();
        valIn.reset();
      }
      IFile.Reader reader = new InMemoryReader(mergeManager, null, bout.getBuffer(), 0,
          bout.getBuffer().length);
      segments.add(new TezMerger.Segment(reader, true));

      data.clear();
      writer.close();
View Full Code Here
83
84
85
86
87
88
89
90
    byte[] bytes = new byte[(int)writer.getRawLength()];

    readDataToMem(inStream, bytes);
    inStream.close();

    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
115
116
117
118
119
120
121
122
    readDataToMem(inStream, bytes);
    inStream.close();


    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
  public IFile.Reader openIFileReader(FetchedInput fetchedInput)
      throws IOException {
    if (fetchedInput.getType() == Type.MEMORY) {
      MemoryFetchedInput mfi = (MemoryFetchedInput) fetchedInput;

      return new InMemoryReader(null, mfi.getInputAttemptIdentifier(),
          mfi.getBytes(), 0, (int) mfi.getActualSize());
    } else {
      return new IFile.Reader(fetchedInput.getInputStream(),
          fetchedInput.getCompressedSize(), codec, null, ifileReadAhead,
          ifileReadAheadLength, ifileBufferSize);
View Full Code Here
83
84
85
86
87
88
89
90
    byte[] bytes = new byte[(int)writer.getRawLength()];

    readDataToMem(inStream, bytes);
    inStream.close();

    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
115
116
117
118
119
120
121
122
    readDataToMem(inStream, bytes);
    inStream.close();


    InMemoryReader inMemReader = new InMemoryReader(null, new InputAttemptIdentifier(0, 0), bytes, 0, bytes.length);
    readAndVerify(inMemReader, data);
  }
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
  public IFile.Reader openIFileReader(FetchedInput fetchedInput)
      throws IOException {
    if (fetchedInput.getType() == Type.MEMORY) {
      MemoryFetchedInput mfi = (MemoryFetchedInput) fetchedInput;

      return new InMemoryReader(null, mfi.getInputAttemptIdentifier(),
          mfi.getBytes(), 0, (int) mfi.getActualSize());
    } else {
      return new IFile.Reader(fetchedInput.getInputStream(),
          fetchedInput.getCompressedSize(), codec, null, null, ifileReadAhead,
          ifileReadAheadLength, ifileBufferSize);
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.shuffle.impl.InMemoryReader

  • org.apache.tez.runtime.library.broadcast.input.BroadcastKVReader
  • org.apache.tez.runtime.library.common.readers.ShuffledUnorderedKVReader
  • org.apache.tez.runtime.library.common.readers.UnorderedKVReader
  • org.apache.tez.runtime.library.common.sort.impl.TestIFile
  • org.apache.tez.runtime.library.common.TestValuesIterator
  • java.io.File
  • java.io.FileOutputStream
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.