Package org.apache.avro.mapred

Examples of org.apache.avro.mapred.FsInput

95
96
97
98
99
100
101
102
103
    /*
     * To be used when used externally
     */
    public void initialize(Path path, Configuration conf) throws IOException {
      FsInput fSInput = new FsInput(path, conf);
      reader = DataFileReader.openReader(fSInput, specificReader);
      reader.sync(0);
    }
View Full Code Here
80
81
82
83
84
85
86
87
88
    avroSchema = AvroUtil.getAvroSchema(meta, conf);
    avroFields = avroSchema.getFields();

    DatumReader<GenericRecord> datumReader =
        new GenericDatumReader<GenericRecord>(avroSchema);
    SeekableInput input = new FsInput(fragment.getPath(), conf);
    dataFileReader = new DataFileReader<GenericRecord>(input, datumReader);
    super.init();
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
    FileSplit fsplit = (FileSplit) isplit;
    start  = fsplit.getStart();
    end    = fsplit.getStart() + fsplit.getLength();   
    reader = DataFileReader.openReader
       (new FsInput(fsplit.getPath(), tc.getConfiguration()),
           new GenericDatumReader<GenericData.Record>(schema));
    reader.sync(start);
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  @Override
  public Iterator<T> read(FileSystem fs, final Path path) {
    this.mapFn.initialize();
    try {
      FsInput fsi = new FsInput(path, fs.getConf());
      final DataFileReader<T> reader = new DataFileReader<T>(fsi, recordReader);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    if(latest != null) {
      gdr.setExpected(latest);
    }

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
    this.recordReaderID = new UID();
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
  private DataFileReader<GenericRecord> read(Path filename) throws IOException {
    Configuration conf = new Configuration();
    if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FsInput fsInput = new FsInput(filename, conf);
    DatumReader<GenericRecord> datumReader =
      new GenericDatumReader<GenericRecord>();
    return new DataFileReader<GenericRecord>(fsInput, datumReader);
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
  private long start;
  private long end;

  public TetherRecordReader(JobConf job, FileSplit split)
    throws IOException {
    this.in = new FsInput(split.getPath(), job);
    this.reader =
      new DataFileReader<Object>(in, new GenericDatumReader<Object>());

    reader.sync(split.getStart());                    // sync to start
    this.start = in.tell();
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context)
      throws IOException, InterruptedException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration conf = context.getConfiguration();
    SeekableInput in = new FsInput(split.getPath(), conf);
    DatumReader<T> datumReader = new GenericDatumReader<T>();
    this.reader = DataFileReader.openReader(in, datumReader);
    reader.sync(split.getStart());                    // sync to start
    this.start = reader.tell();
    this.end = split.getStart() + split.getLength();
View Full Code Here
206
207
208
209
210
211
212
213
214
215
  private DataFileReader<GenericRecord> read(Path filename) throws IOException {
    Configuration conf = new Configuration();
    if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FsInput fsInput = new FsInput(filename, conf);
    DatumReader<GenericRecord> datumReader =
      new GenericDatumReader<GenericRecord>();
    return new DataFileReader<GenericRecord>(fsInput, datumReader);
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
  @Override
  public Iterator<T> read(FileSystem fs, final Path path) {
    this.mapFn.initialize();
    try {
      FsInput fsi = new FsInput(path, fs.getConf());
      final DataFileReader<T> reader = new DataFileReader<T>(fsi, recordReader);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here
TOP

Related Classes of org.apache.avro.mapred.FsInput

  • com.cloudera.sqoop.TestAvroImport
  • com.cloudera.sqoop.testutil.LobAvroImportTestCase
  • com.datasalt.pangool.tuplemr.avro.AvroRecordReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.linkedin.camus.sweeper.mapreduce.CamusSweeperAvroKeyJob
  • com.linkedin.haivvreo.AvroGenericRecordReader
  • com.linkedin.pig.FastAvroRecordReader
  • MaxWidgetIdGenericAvro
  • org.apache.avro.hadoop.file.SortedKeyValueFile$Reader
  • org.apache.avro.hadoop.file.TestSortedKeyValueFile
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.