Examples of FileSplit

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

192
193
194
195
196
197
198
199
200
201
202
203
     
      ByteRange ret = new ByteRange(val);
     
      InputSplit inSplit = context.getInputSplit();
      if(inSplit instanceof FileSplit) {
        FileSplit fs = (FileSplit) inSplit;
        fname = fs.getPath().getName();
      }
      ret.split = fname;
     
      if(!valid) {
        context.getCounter("app", "badchunks").increment(1);
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  @Override
  public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
      InterruptedException {

    FileSplit fSplit = (FileSplit)split;
    Path path = fSplit.getPath();
    Configuration conf = context.getConfiguration();
    this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf);
    this.end = fSplit.getStart() + fSplit.getLength();

    if(fSplit.getStart() > in.getPosition()) {
      in.sync(fSplit.getStart());
    }

    this.start = in.getPosition();
    more = start < end;
View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    @SuppressWarnings("deprecation")
    @Override
    public void initialize(InputSplit split, TaskAttemptContext ctx)
    throws IOException, InterruptedException {

        FileSplit fileSplit = (FileSplit)split;
        Configuration conf = ctx.getConfiguration();
        splitPath = fileSplit.getPath();

        rcFileRecordReader = new RCFileRecordReader<LongWritable, BytesRefArrayWritable>(conf,
                new org.apache.hadoop.mapred.FileSplit(splitPath, fileSplit.getStart(), fileSplit.getLength(),
                        new org.apache.hadoop.mapred.JobConf(conf)) );


        key = rcFileRecordReader.createKey();
        value = rcFileRecordReader.createValue();
View Full Code Here

Examples of org.apache.hama.bsp.FileSplit

38
39
40
41
42
43
44
45
46
47
48
  @Test
  public void testFileRead() throws Exception {
    VectorDoubleFileInputFormat inputFormat = new VectorDoubleFileInputFormat();
    Path file = new Path("src/test/resources/vd_file_sample.txt");
    InputSplit split = new FileSplit(file, 0, 1000, new String[]{"localhost"});
    BSPJob job = new BSPJob();
    RecordReader<VectorWritable, DoubleWritable> recordReader = inputFormat.getRecordReader(split, job);
    assertNotNull(recordReader);
    VectorWritable key = recordReader.createKey();
    assertNotNull(key);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.