Package org.apache.avro.mapred

Examples of org.apache.avro.mapred.FsInput

51
52
53
54
55
56
57
58
59
60
61
  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration conf = context.getConfiguration();
    SeekableInput in = new FsInput(split.getPath(), conf);
    DatumReader<T> datumReader = null;
    if (context.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true)) {
      ReflectDataFactory factory = Avros.getReflectDataFactory(conf);
      datumReader = factory.getReader(schema);
    } else {
View Full Code Here
72
73
74
75
76
77
78
79
80
81
    if(latest != null) {
      gdr.setExpected(latest);
    }

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
  }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
    /*
     * To be used when used externally
     */
    public void initialize(Path path, Configuration conf) throws IOException {
      FsInput fSInput = new FsInput(path, conf);
      reader = DataFileReader.openReader(fSInput, specificReader);
      end = Long.MAX_VALUE;
      reader.sync(0);
      start = reader.tell();
    }
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
        if (schema != null)
          return schema;
      }
      else if (f.getPath().getName().endsWith(".avro"))
      {
        FsInput fi = new FsInput(f.getPath(), fs.getConf());
        GenericDatumReader<GenericRecord> genReader = new GenericDatumReader<GenericRecord>();
        DataFileReader<GenericRecord> reader = new DataFileReader<GenericRecord>(fi, genReader);
        return reader.getSchema();
      }
    }
View Full Code Here
166
167
168
169
170
171
172
   * @param path The path to the avro container file.
   * @throws IOException If there is an error reading from the path.
   */
  protected SeekableInput createSeekableInput(Configuration conf, Path path)
      throws IOException {
    return new FsInput(path, conf);
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
  }
 
  public void initialize(FileSplit split,Configuration conf)
      throws IOException, InterruptedException{
      init(DataFileReader.openReader
        (new FsInput(split.getPath(), conf),
         isReflect
         ? new ReflectDatumReader<T>(schema)
         : new SpecificDatumReader<T>(schema)),
        split);
  }
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
    /*
     * To be used when used externally
     */
    public void initialize(Path path, Configuration conf) throws IOException {
      FsInput fSInput = new FsInput(path, conf);
      reader = DataFileReader.openReader(fSInput, specificReader);
      end = Long.MAX_VALUE;
      reader.sync(0);
      start = reader.tell();
    }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
  @Override
  public Iterator<T> read(FileSystem fs, final Path path) {
    this.mapFn.setConfigurationForTest(conf);
    this.mapFn.initialize();
    try {
      FsInput fsi = new FsInput(path, fs.getConf());
      final DataFileReader<T> reader = new DataFileReader<T>(fsi, recordReader);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here
60
61
62
63
64
65
66
67
68
69
    GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();

    if(latest != null) gdr.setExpected(latest);

    this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr);
    this.reader.sync(split.getStart());
    this.start = reader.tell();
    this.stop = split.getStart() + split.getLength();
  }
View Full Code Here
96
97
98
99
100
101
102
   * @param filename The filename to be opened
   * @throws IOException
   */
  static FsInput openSeekableFromFS(String filename)
      throws IOException {      
    return new FsInput(new Path(filename), new Configuration());
  }
View Full Code Here
TOP

Related Classes of org.apache.avro.mapred.FsInput

  • com.cloudera.sqoop.TestAvroImport
  • com.cloudera.sqoop.testutil.LobAvroImportTestCase
  • com.datasalt.pangool.tuplemr.avro.AvroRecordReader
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.linkedin.camus.sweeper.mapreduce.CamusSweeperAvroKeyJob
  • com.linkedin.haivvreo.AvroGenericRecordReader
  • com.linkedin.pig.FastAvroRecordReader
  • MaxWidgetIdGenericAvro
  • org.apache.avro.hadoop.file.SortedKeyValueFile$Reader
  • org.apache.avro.hadoop.file.TestSortedKeyValueFile
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.