Package org.apache.avro.hadoop.io

Examples of org.apache.avro.hadoop.io.AvroDatumConverterFactory

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    // Use a writer to generate a Avro container file in memory.
    // Write two records: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    R1 record = new R1();
    record.attribute = "test";
    AvroValue<R1> avroValue = new AvroValue<R1>(record);

    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    AvroDatumConverterFactory factory =
      new AvroDatumConverterFactory(job.getConfiguration());

    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);

    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<R1>, R1> valueConverter =
      factory.create((Class<AvroValue<R1>>) avroValue.getClass());

    AvroKeyValueRecordWriter<Text, AvroValue<R1>> writer =
      new AvroKeyValueRecordWriter<Text, AvroValue<R1>>(
        keyConverter, valueConverter, new ReflectData(),
        CodecFactory.nullCodec(), outputStream);
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));

    // Write a marker followed by each record: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
    AvroKeyValueRecordWriter<Text, AvroValue<TextStats>> writer
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
  @Override
  @SuppressWarnings("unchecked")
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();

    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(conf);

    AvroDatumConverter<K, ?> keyConverter = converterFactory.create(
        (Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create(
        (Class<V>) context.getOutputValueClass());

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    // Use a writer to generate a Avro container file in memory.
    // Write two records: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    R1 record = new R1();
    record.attribute = "test";
    AvroValue<R1> avroValue = new AvroValue<R1>(record);

    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    AvroDatumConverterFactory factory =
      new AvroDatumConverterFactory(job.getConfiguration());

    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);

    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<R1>, R1> valueConverter =
      factory.create((Class<AvroValue<R1>>) avroValue.getClass());

    AvroKeyValueRecordWriter<Text, AvroValue<R1>> writer =
      new AvroKeyValueRecordWriter<Text, AvroValue<R1>>(keyConverter,
        valueConverter, CodecFactory.nullCodec(), outputStream);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    // Use a writer to generate a Avro container file in memory.
    // Write two records: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
public class AvroKeyValueOutputFormat<K, V> extends AvroOutputFormatBase<K, V> {
  /** {@inheritDoc} */
  @Override
  @SuppressWarnings("unchecked")
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(
        context.getConfiguration());

    AvroDatumConverter<K, ?> keyConverter = converterFactory.create(
        (Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create(
        (Class<V>) context.getOutputValueClass());

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
        getCompressionCodec(context), getAvroFileOutputStream(context));
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
public class AvroKeyValueOutputFormat<K, V> extends AvroOutputFormatBase<K, V> {
  /** {@inheritDoc} */
  @Override
  @SuppressWarnings("unchecked")
  public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException {
    AvroDatumConverterFactory converterFactory = new AvroDatumConverterFactory(
        context.getConfiguration());

    AvroDatumConverter<K, ?> keyConverter = converterFactory.create(
        (Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create(
        (Class<V>) context.getOutputValueClass());

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
        getCompressionCodec(context), getAvroFileOutputStream(context));
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    AvroJob.setOutputValueSchema(job, TextStats.SCHEMA$);
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);

    AvroDatumConverterFactory factory = new AvroDatumConverterFactory(job.getConfiguration());
    AvroDatumConverter<Text, ?> keyConverter = factory.create(Text.class);
    AvroValue<TextStats> avroValue = new AvroValue<TextStats>(null);
    @SuppressWarnings("unchecked")
    AvroDatumConverter<AvroValue<TextStats>, ?> valueConverter
        = factory.create((Class<AvroValue<TextStats>>) avroValue.getClass());
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

    // Use a writer to generate a Avro container file in memory.
    // Write two records: <'apple', TextStats('apple')> and <'banana', TextStats('banana')>.
View Full Code Here
TOP

Related Classes of org.apache.avro.hadoop.io.AvroDatumConverterFactory

  • com.cloudera.recordbreaker.analyzer.SequenceFileSchemaDescriptor
  • com.cloudera.recordbreaker.hive.SequenceFileSerDe
  • org.apache.avro.mapreduce.AvroKeyValueOutputFormat
  • org.apache.avro.mapreduce.TestAvroKeyValueRecordWriter
  • org.apache.avro.Schema
  • org.apache.trevni.avro.mapreduce.AvroTrevniKeyValueRecordWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.