Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

Class org.apache.hadoop.fs.FSDataInputStream

Examples of org.apache.hadoop.fs.FSDataInputStream.seek()

org.apache.hadoop.fs.FSDataInputStream.seek()
Seek to the given offset. @param desired offset to seek to

         * send it to the reducer.
         */
        //open the map-output file
        mapOutputIn = fileSys.open(mapOutputFileName);
        //seek to the correct offset for the reduce
        mapOutputIn.seek(startOffset);
          
        long totalRead = 0;
        int len = mapOutputIn.read(buffer, 0,
                                   partLength < MAX_BYTES_TO_READ 
                                   ? (int)partLength : MAX_BYTES_TO_READ);

View Full Code Here

        for (int parts = 0; parts < partitions; parts++){
          List<SegmentDescriptor> segmentList =
            new ArrayList<SegmentDescriptor>(numSpills);
          for(int i = 0; i < numSpills; i++) {
            FSDataInputStream indexIn = localFs.open(indexFileName[i]);
            indexIn.seek(parts * 16);
            long segmentOffset = indexIn.readLong();
            long segmentLength = indexIn.readLong();
            indexIn.close();
            SegmentDescriptor s = sorter.new SegmentDescriptor(segmentOffset,
                                                               segmentLength, filename[i]);

View Full Code Here

         * send it to the reducer.
         */
        //open the map-output file
        mapOutputIn = fileSys.open(mapOutputFileName);
        //seek to the correct offset for the reduce
        mapOutputIn.seek(startOffset);
          
        long totalRead = 0;
        int len = mapOutputIn.read(buffer, 0,
                                   partLength < MAX_BYTES_TO_READ 
                                   ? (int)partLength : MAX_BYTES_TO_READ);

View Full Code Here

        InputStream input = fs.open(p);
        holder.setStream(input);
      } else {
        // no codec, with split
        FSDataInputStream input = fs.open(inputPath);
        input.seek(getSplit().getStart());
        holder.setStream(input);
      }
    } else {
      Class<?> clazz = ClassUtils.resolveClassName(getCodec().getCodecClass(), getClass().getClassLoader());
      CompressionCodec compressionCodec = (CompressionCodec) ReflectionUtils.newInstance(clazz,

View Full Code Here

        for (Path src : FileUtil.stat2Paths(srcFs.globStatus(srcPat), srcPat)) {
          Assert.isTrue(srcFs.isFile(src), "Source must be a file");
          i = srcFs.open(src);
          switch (i.readShort()) {
          case 0x1f8b: // RFC 1952
            i.seek(0);
            in = new GZIPInputStream(i);
            break;
          case 0x5345: // 'S' 'E'
            if (i.readByte() == 'Q') {
              i.close();

View Full Code Here

            break;
          default:
            in = i;
            break;
          }
          i.seek(0);
          texts.add(getContent(in));
        }
      } catch (IOException ex) {
        throw new HadoopException("Cannot read " + uri + ";" + ex.getMessage(), ex);
      } finally {

View Full Code Here

            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(job);
            FSDataInputStream fileIn = fs.open(split.getPath());
            
            // Seek to the start of the file
            fileIn.seek(start);
        
            if(file.toString().endsWith(".bz2") || file.toString().endsWith(".bz"))
            {
              // For bzip2 files use CBZip2InputStream to read and supply the upper input stream.
               CBZip2InputStream in = new CBZip2InputStream(fileIn,9, end);

View Full Code Here

      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;
        fileIn.seek(start);
      }
      in = new LineReader(fileIn, job);
    }
    if (skipFirstLine) { // skip first line and re-establish "start".
      start += in.readLine(new Text(), 0,

View Full Code Here

  @SuppressWarnings("unchecked")
  private <T> T getSplitDetails(Path file, long offset)
   throws IOException {
    FileSystem fs = file.getFileSystem(conf);
    FSDataInputStream inFile = fs.open(file);
    inFile.seek(offset);
    String className = Text.readString(inFile);
    Class<T> cls;
    try {
      cls = (Class<T>) conf.getClassByName(className);
    } catch (ClassNotFoundException ce) {

View Full Code Here

                Counter counter) throws IOException, InterruptedException {
            FileSystem fs = FileSystem.get(path.toUri(), getConf());
            FSDataInputStream in = fs.open(path);
            boolean succeed = false;
            try {
                in.seek(offset);
                ModelInput<StringBuilder> result = format.createInput(
                        dataType,
                        path.toString(),
                        new CountInputStream(in, counter),
                        offset,

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.