Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

180
181
182
183
184
185
186
187
188
189
190
          BlockInfo bi = this.stripeBlocks.get(startOffset + i);
          FileStatus curFile = lfs.get(bi.fileIdx);
          long seekOffset = bi.blockId * curFile.getBlockSize();
          Path srcFile = curFile.getPath();
          FSDataInputStream in = fs.open(srcFile, bufferSize);
          in.seek(seekOffset);
          LOG.info("Opening stream at " + srcFile + ":" + seekOffset);
          blocks[i] = in;
        } else {
          LOG.info("Using zeros at block " + i);
          // We have no src data at this offset.
View Full Code Here
265
266
267
268
269
270
271
272
273
274
275
      } else {
        LOG.info("Opening " + fstat.getPath() + ":" + offset +
                 " for location " + locationIndex);
        FSDataInputStream s = fs.open(
            fstat.getPath(), conf.getInt("io.file.buffer.size", 64 * 1024));
        s.seek(offset);
        return s;
      }
    }
  }

View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
        HadoopThriftHandler.LOG.debug("read: " + tout.id +
                                     " offset: " + offset +
                                     " length: " + length);
        FSDataInputStream in = (FSDataInputStream)lookup(tout.id);
        if (in.getPos() != offset) {
          in.seek(offset);
        }
        byte[] tmp = new byte[length];
        int numbytes = in.read(offset, tmp, 0, length);
        HadoopThriftHandler.LOG.debug("read done: " + tout.id);
        return new String(tmp, 0, numbytes, "UTF-8");
View Full Code Here
221
222
223
224
225
226
227
228
229
    assert(offset < parityStat.getLen());
    LOG.info("Opening " + parityFile + ":" + offset +
      " for location " + locationIndex);
    FSDataInputStream s = parityFs.open(
      parityFile, conf.getInt("io.file.buffer.size", 64 * 1024));
    s.seek(offset);
    return s;
  }
}
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    FSDataInputStream in = fs.open(file);
    try {
      for (int idx: idxs) {
        long offset = idx * blockSize;
        LOG.info("Reporting corrupt block " + file + ":" + offset);
        in.seek(offset);
        try {
          in.readFully(new byte[(int)blockSize]);
          fail("Expected exception not thrown for " + file + ":" + offset);
        } catch (org.apache.hadoop.fs.ChecksumException e) {
        } catch (org.apache.hadoop.fs.BlockMissingException bme) {
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
    }

    private void init(Counters.Counter readsCounter) throws IOException {
      if (reader == null) {
        FSDataInputStream in = fs.open(file);
        in.seek(segmentOffset);
        reader = new Reader<K, V>(conf, in, segmentLength, codec, readsCounter);
      }
    }
   
    DataInputBuffer getKey() { return key; }
View Full Code Here
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
         */
        //open the map-output file
        mapOutputIn = rfs.open(mapOutputFileName);

        //seek to the correct offset for the reduce
        mapOutputIn.seek(info.startOffset);
        long rem = info.partLength;
        int len =
          mapOutputIn.read(buffer, 0, (int)Math.min(rem, MAX_BYTES_TO_READ));
        while (rem > 0 && len >= 0) {
          rem -= len;
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
      long bytesAlreadyCopied = 0;
      if (offset != errorOffset) {
        try {
          in = fs.open(
            srcFile, conf.getInt("io.file.buffer.size", 64 * 1024));
          in.seek(offset);
          RaidUtils.copyBytes(in, out, readBufs[0], limit);
          assert(out.getPos() == offset +limit);
          LOG.info("Copied till " + out.getPos() + " from " + srcFile);
          continue;
        } catch (BlockMissingException e) {
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
            FileSystem fs1 = getUnderlyingFileSystem(conf);
            fs1.initialize(npath.toUri(), conf);
            LOG.info("Opening alternate path " + npath + " at offset " + curpos);
            FSDataInputStream fd = fs1.open(npath, buffersize);
            fd.seek(curpos);
            underLyingStream.close();
            underLyingStream = fd;
            lfs.fs = fs1;
            path = npath;
            return;
View Full Code Here
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
            long segmentOffset = indexRecord.startOffset;
            long rawSegmentLength = indexRecord.rawLength;
            long segmentLength = indexRecord.partLength;

            FSDataInputStream in = rfs.open(filename[i]);
            in.seek(segmentOffset);

            Segment<K, V> s =
              new Segment<K, V>(new Reader<K, V>(job, in, segmentLength, codec),
                                true);
            segmentList.add(i, s);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.