Package org.apache.trevni.avro

Examples of org.apache.trevni.avro.HadoopInput

45
46
47
48
49
50
51
  private int [] projectionMap;
  private ColumnValues [] columns;

  public TrevniScanner(Configuration conf, TableMeta meta, Fragment fragment) throws IOException {
    super(conf, meta, fragment);
    reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf));
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
          public void initialize(final InputSplit isplit,
              final TaskAttemptContext tac)
                  throws IOException, InterruptedException {
            fsplit = (FileSplit) isplit;
            params = new AvroColumnReader.Params(
                new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
            Schema inputSchema = getInputAvroSchema();
            params.setSchema(inputSchema);
            reader = new AvroColumnReader<GenericData.Record>(params);
            rows = reader.getRowCount();
          }
View Full Code Here
331
332
333
334
335
336
337
338
339
340
341
      throw new IOException("No path matches pattern " + p.toString());
    }

    AvroColumnReader.Params params =
        new AvroColumnReader.Params(
            new HadoopInput(filePath, job.getConfiguration()));
    AvroColumnReader<GenericData.Record> reader =
        new AvroColumnReader<GenericData.Record>(params);
    Schema s = reader.getFileSchema();
    reader.close();
    return s;
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
          public void initialize(final InputSplit isplit,
              final TaskAttemptContext tac)
                  throws IOException, InterruptedException {
            fsplit = (FileSplit) isplit;
            params = new AvroColumnReader.Params(
                new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
            Schema inputSchema = getInputAvroSchema();
            params.setSchema(inputSchema);
            reader = new AvroColumnReader<GenericData.Record>(params);
            rows = reader.getRowCount();
          }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  @Override
  public Iterator<T> read(FileSystem fs, final Path path) {
    this.mapFn.initialize();
    try {
      HadoopInput input = new HadoopInput(path, fs.getConf());
      final AvroColumnReader<T> reader = getReader(input, aType, schema);
      return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {
        @Override
        public boolean hasNext() {
          return reader.hasNext();
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
          public void initialize(final InputSplit isplit,
              final TaskAttemptContext tac)
                  throws IOException, InterruptedException {
            fsplit = (FileSplit) isplit;
            params = new AvroColumnReader.Params(
                new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
            Schema inputSchema = getInputAvroSchema();
            params.setSchema(inputSchema);
            reader = new AvroColumnReader<GenericData.Record>(params);
            rows = reader.getRowCount();
          }
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
      throw new IOException("No path matches pattern " + p.toString());
    }

    AvroColumnReader.Params params =
        new AvroColumnReader.Params(
            new HadoopInput(filePath, job.getConfiguration()));
    AvroColumnReader<GenericData.Record> reader =
        new AvroColumnReader<GenericData.Record>(params);
    Schema s = reader.getFileSchema();
    reader.close();
    return s;
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
          public void initialize(final InputSplit isplit,
              final TaskAttemptContext tac)
                  throws IOException, InterruptedException {
            fsplit = (FileSplit) isplit;
            params = new AvroColumnReader.Params(
                new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
            Schema inputSchema = getInputAvroSchema();
            params.setSchema(inputSchema);
            reader = new AvroColumnReader<GenericData.Record>(params);
            rows = reader.getRowCount();
          }
View Full Code Here
45
46
47
48
49
50
51
  private int [] projectionMap;
  private ColumnValues [] columns;

  public TrevniScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException {
    super(conf, schema, meta, fragment);
    reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf));
  }
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
            @Override
            public void initialize(InputSplit isplit, TaskAttemptContext tac) throws IOException, InterruptedException {
              // TODO Auto-generated method stub
              fsplit = (FileSplit) isplit;
              params = new AvroColumnReader.Params(
                  new HadoopInput(fsplit.getPath(), tac.getConfiguration()));
              Schema inputSchema = getInputAvroSchema();
              System.err.printf("initializing RecordReader with schema %s\n", inputSchema);
              params.setSchema(inputSchema);
              reader = new AvroColumnReader<GenericData.Record>(params);
              rows = reader.getRowCount();
View Full Code Here
TOP

Related Classes of org.apache.trevni.avro.HadoopInput

  • com.linkedin.pig.TrevniStorage
  • com.linkedin.pig.TrevniStorageInputFormat
  • org.apache.avro.tool.TrevniUtil
  • org.apache.crunch.io.avro.trevni.TrevniFileReaderFactory
  • org.apache.pig.builtin.TrevniStorage
  • org.apache.pig.builtin.TrevniStorageInputFormat
  • org.apache.tajo.storage.trevni.TrevniScanner
  • org.apache.trevni.avro.mapreduce.AvroTrevniRecordReaderBase
  • org.apache.trevni.tool.Util
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.