Package org.apache.hadoop.mapreduce.lib.input

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

268
269
270
271
272
273
274
275
276
277
278
        long pos = 0;
        int n;
        try {
          while ((n = reader.readLine(key)) > 0) {
            String[] hosts = getStoreDirHosts(fs, path);
            splits.add(new FileSplit(path, pos, n, hosts));
            pos += n;
          }
        } finally {
          reader.close();
        }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    private FSDataInputStream fileIn = null;
    private static InterSedes sedes = InterSedesFactory.getInterSedesInstance();

    public void initialize(InputSplit genericSplit, TaskAttemptContext context)
                    throws IOException {
        FileSplit split = (FileSplit) genericSplit;
        Configuration job = context.getConfiguration();
        start = split.getStart();
        end = start + split.getLength();
        final Path file = split.getPath();

        // open the file and seek to the start of the split
        FileSystem fs = file.getFileSystem(job);
        fileIn = fs.open(split.getPath());
        reader = new Reader(fileIn, fs.getFileStatus(file).getLen(), job);
        scanner = reader.createScannerByByteRange(start, split.getLength());
    }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
  public static final int RECORD_3 = 0x03;
  private DataInputStream inData = null;

  public void initialize(InputSplit genericSplit,
                         TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    if (start != 0) {
        fileIn.seek(start);
    }
    in = new BufferedPositionedInputStream(fileIn, start);
    inData = new DataInputStream(in);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
  private DataInputStream inData = null;
  private static InterSedes sedes = InterSedesFactory.getInterSedesInstance();

  public void initialize(InputSplit genericSplit,
                         TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    if (start != 0) {
        fileIn.seek(start);
    }
    in = new BufferedPositionedInputStream(fileIn, start);
    inData = new DataInputStream(in);
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
     
      ByteRange ret = new ByteRange(val);
     
      InputSplit inSplit = context.getInputSplit();
      if(inSplit instanceof FileSplit) {
        FileSplit fs = (FileSplit) inSplit;
        fname = fs.getPath().getName();
      }
      ret.split = fname;
     
      if(!valid) {
        context.getCounter("app", "badchunks").increment(1);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  @Override
  public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
      InterruptedException {

    FileSplit fSplit = (FileSplit)split;
    Path path = fSplit.getPath();
    Configuration conf = context.getConfiguration();
    this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf);
    this.end = fSplit.getStart() + fSplit.getLength();

    if(fSplit.getStart() > in.getPosition()) {
      in.sync(fSplit.getStart());
    }

    this.start = in.getPosition();
    more = start < end;
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    @SuppressWarnings("deprecation")
    @Override
    public void initialize(InputSplit split, TaskAttemptContext ctx)
    throws IOException, InterruptedException {

        FileSplit fileSplit = (FileSplit)split;
        Configuration conf = ctx.getConfiguration();
        splitPath = fileSplit.getPath();

        rcFileRecordReader = new RCFileRecordReader<LongWritable, BytesRefArrayWritable>(conf,
                new org.apache.hadoop.mapred.FileSplit(splitPath, fileSplit.getStart(), fileSplit.getLength(),
                        new org.apache.hadoop.mapred.JobConf(conf)) );


        key = rcFileRecordReader.createKey();
        value = rcFileRecordReader.createValue();
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
      @Override
      public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {
        if (filePath == null) {
          FileSplit split = (FileSplit) context.getInputSplit();
          filePath = split.getPath().toString();
        }
        String line = value.toString();
        StringTokenizer st = new StringTokenizer(line, " ");
        while (st.hasMoreElements()) {
          byte[] word = st.nextToken().getBytes();
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
public abstract class FileInputLoadFunc extends LoadFunc implements OrderedLoadFunc  {
   
    @Override
    public WritableComparable<?> getSplitComparable(InputSplit split)
    throws IOException{
        FileSplit fileSplit = null;
        if(split instanceof FileSplit){
            fileSplit = (FileSplit)split;
        }else{
            throw new RuntimeException("LoadFunc expected split of type FileSplit");
        }
       
        return new FileSplitComparable(
                fileSplit.getPath().toString(),
                fileSplit.getStart()
        );
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
    private long count;
    private boolean seeked = false;

    @Override
    public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
      FileSplit fileSplit = (FileSplit) split;
      conf = context.getConfiguration();
      Path path = fileSplit.getPath();
      FileSystem fs = path.getFileSystem(conf);
      LOG.info("Initialize HFileRecordReader for " + path);
      this.in = HFile.createReader(fs, path, new CacheConfig(conf), conf);

      // The file info must be loaded before the scanner can be used.
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.lib.input.FileSplit

  • com.asakusafw.runtime.stage.input.TemporaryInputFormat
  • com.asakusafw.runtime.stage.input.TemporaryInputFormatTest
  • com.chine.kmeans.mapreduce.dataprep.DataPrepMapper
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.hadoop.mapreduce.LzoLineRecordReader
  • com.hadoop.mapreduce.LzoSplitRecordReader
  • com.hadoop.mapreduce.LzoTextInputFormat
  • com.mongodb.hadoop.splitter.BSONSplitter
  • com.thinkaurelius.faunus.formats.graphson.GraphSONRecordReaderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.