Package org.apache.hadoop.mapreduce.lib.input

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

85
86
87
88
89
90
91
92
93
94
95
    List<InputSplit> splits = new ArrayList<InputSplit>(nSplits);
   
    for (int i=0; i< nSplits; ++i) {
      TaskID taskId = new TaskID(jobContext.getJobID(), TaskType.MAP, i);
      chunks.get(i).assignTo(taskId);
      splits.add(new FileSplit(chunks.get(i).getPath(), 0,
          // Setting non-zero length for FileSplit size, to avoid a possible
          // future when 0-sized file-splits are considered "empty" and skipped
          // over.
          getMinRecordsPerChunk(jobContext.getConfiguration()),
          null));
View Full Code Here
152
153
154
155
156
157
158
159
160
  }

  private void openForRead(TaskAttemptContext taskAttemptContext)
          throws IOException, InterruptedException {
    reader = new SequenceFileRecordReader<K, V>();
    reader.initialize(new FileSplit(chunkFilePath, 0,
            DistCpUtils.getFileSize(chunkFilePath, configuration), null),
            taskAttemptContext);
  }
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    long lastEnd = 0;

    //Verify if each split's start is matching with the previous end and
    //we are not missing anything
    for (InputSplit split : splits) {
      FileSplit fileSplit = (FileSplit) split;
      long start = fileSplit.getStart();
      Assert.assertEquals(lastEnd, start);
      lastEnd = start + fileSplit.getLength();
    }

    //Verify there is nothing more to read from the input file
    SequenceFile.Reader reader
            = new SequenceFile.Reader(cluster.getFileSystem().getConf(),
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
      reader = getListingFileReader(configuration);
      while (reader.next(srcRelPath, srcFileStatus)) {
        // If adding the current file would cause the bytes per map to exceed
        // limit. Add the current file to new split
        if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) {
          FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
              lastPosition - lastSplitStart, null);
          if (LOG.isDebugEnabled()) {
            LOG.debug ("Creating split : " + split + ", bytes in split: " + currentSplitSize);
          }
          splits.add(split);
          lastSplitStart = lastPosition;
          currentSplitSize = 0;
        }
        currentSplitSize += srcFileStatus.getLen();
        lastPosition = reader.getPosition();
      }
      if (lastPosition > lastSplitStart) {
        FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
            lastPosition - lastSplitStart, null);
        if (LOG.isDebugEnabled()) {
          LOG.info ("Creating split : " + split + ", bytes in split: " + currentSplitSize);
        }
        splits.add(split);
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    public KryoRecordReader() {
    }

    @Override
    public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException {
        final FileSplit split = (FileSplit) genericSplit;
        final Configuration job = context.getConfiguration();
        long start = split.getStart();
        final Path file = split.getPath();
        if (null != new CompressionCodecFactory(job).getCodec(file)) {
            throw new IllegalStateException("Compression is not supported for the (binary) Gremlin Kryo format");
        }
        // open the file and seek to the start of the split
        this.inputStream = file.getFileSystem(job).open(split.getPath());
        this.inputStream.seek(start);
        final long newStart = seekToHeader(this.inputStream, start);
        this.vertexStreamIterator = new VertexStreamIterator(this.inputStream, split.getLength() - (newStart - start), KryoReader.build().create());
    }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
      return progress;
    }
   
    @Override
    public void initialize(InputSplit is, TaskAttemptContext context) throws IOException, InterruptedException {
      FileSplit fileSplit = (FileSplit) is;
     
      Configuration conf = new Configuration();
      FileSystem fs = FileSystem.get(conf);
     
      key = new LogFileKey();
      value = new LogFileValue();
     
      fsdis = fs.open(fileSplit.getPath());
      FileStatus status = fs.getFileStatus(fileSplit.getPath());
      length = status.getLen();
    }
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
228
    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      if (first) {
        FileSplit split = (FileSplit) context.getInputSplit();
        Path path = split.getPath(); // current split path
        lvalue.set(path.getName());
        lkey.set(key.get());
        context.write(lkey, lvalue);

        first = false;
View Full Code Here
377
378
379
380
381
382
383
384
385
      private final FileSplit fileSplit;

      @SuppressWarnings("unused")
      public SequenceFileRecordReaderWrapper(CombineFileSplit split, TaskAttemptContext context, Integer index)
          throws IOException {
        fileSplit = new FileSplit(split.getPath(index), split.getOffset(index), split.getLength(index),
            split.getLocations());
        delegate = new SequenceFileInputFormat<Writable, Text>().createRecordReader(fileSplit, context);
      }
View Full Code Here
366
367
368
369
370
371
372
373
374
375
376
  }

  protected Path getCurrentFile(Context context) throws IOException {
    InputSplit split = context.getInputSplit();
    if (split != null && split instanceof FileSplit) {
      FileSplit inputSplit = (FileSplit) split;
      Path path = inputSplit.getPath();
      return path.makeQualified(path.getFileSystem(context.getConfiguration()));
    }
    return null;
  }
View Full Code Here
359
360
361
362
363
364
365
366
367
368
      private final FileSplit fileSplit;

    @SuppressWarnings("unused")
    public SequenceFileRecordReaderWrapper(CombineFileSplit split,
            TaskAttemptContext context, Integer index) throws IOException{
            fileSplit = new FileSplit(split.getPath(index),
                      split.getOffset(index), split.getLength(index),
                      split.getLocations());
            delegate = new SequenceFileInputFormat<Writable,Text>().createRecordReader(fileSplit, context);
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.input.FileSplit

  • com.asakusafw.runtime.stage.input.TemporaryInputFormat
  • com.asakusafw.runtime.stage.input.TemporaryInputFormatTest
  • com.chine.kmeans.mapreduce.dataprep.DataPrepMapper
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.hadoop.mapreduce.LzoTextInputFormat
  • com.mongodb.hadoop.splitter.BSONSplitter
  • com.thinkaurelius.faunus.formats.graphson.GraphSONRecordReaderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.