Package org.apache.avro.generic

Examples of org.apache.avro.generic.GenericData

149
150
151
152
153
154
155
156
157
158
159
160
161
162
    boolean isMapOnly = job.getNumReduceTasks() == 0;
    Schema schema = isMapOnly
      ? AvroJob.getMapOutputSchema(job)
      : AvroJob.getOutputSchema(job);
    GenericData dataModel = AvroJob.createDataModel(job);

    final DataFileWriter<T> writer =
      new DataFileWriter<T>(dataModel.createDatumWriter(null));
   
    configureDataFileWriter(writer, job);

    Path path = FileOutputFormat.getTaskOutputPath(job, name+EXT);
    writer.create(schema, path.getFileSystem(job).create(path));
View Full Code Here
109
110
111
112
113
114
115
116
  }

  @Test public void testUnionWithCollection() {
    Schema s = new Schema.Parser().parse
      ("[\"null\", {\"type\":\"array\",\"items\":\"float\"}]");
    GenericData data = ReflectData.get();
    assertEquals(1, data.resolveUnion(s, new ArrayList<Float>()));
  }
View Full Code Here
116
117
118
119
120
121
122
123
  }

  @Test public void testUnionWithMap() {
    Schema s = new Schema.Parser().parse
      ("[\"null\", {\"type\":\"map\",\"values\":\"float\"}]");
    GenericData data = ReflectData.get();
    assertEquals(1, data.resolveUnion(s, new HashMap<String,Float>()));
  }
View Full Code Here
124
125
126
127
128
129
130
131
  @Test public void testUnionWithFixed() {
    Schema s = new Schema.Parser().parse
        ("[\"null\", {\"type\":\"fixed\",\"name\":\"f\",\"size\":1}]");
    Schema f = new Schema.Parser().parse("{\"type\":\"fixed\",\"name\":\"f\",\"size\":1}");
    GenericData data = ReflectData.get();
    assertEquals(1, data.resolveUnion(s, new GenericData.Fixed(f)));
  }
View Full Code Here
132
133
134
135
136
137
138
139
  @Test public void testUnionWithEnum() {
    Schema s = new Schema.Parser().parse
        ("[\"null\", {\"type\":\"enum\",\"name\":\"E\",\"namespace\":" +
            "\"org.apache.avro.reflect.TestReflect$\",\"symbols\":[\"A\",\"B\"]}]");
    GenericData data = ReflectData.get();
    assertEquals(1, data.resolveUnion(s, E.A));
  }
View Full Code Here
138
139
140
141
142
143
144
145
    assertEquals(1, data.resolveUnion(s, E.A));
  }

  @Test public void testUnionWithBytes() {
    Schema s = new Schema.Parser().parse ("[\"null\", \"bytes\"]");
    GenericData data = ReflectData.get();
    assertEquals(1, data.resolveUnion(s, ByteBuffer.wrap(new byte[]{1})));
  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
public class TestAvroKeyRecordWriter {
  @Test
  public void testWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  }
 
  @Test
  public void testSycnableWrite() throws IOException {
    Schema writerSchema = Schema.create(Schema.Type.INT);
    GenericData dataModel = new ReflectData();
    CodecFactory compressionCodec = CodecFactory.nullCodec();
    FileOutputStream outputStream = new FileOutputStream(new File("target/temp.avro"));
    TaskAttemptContext context = createMock(TaskAttemptContext.class);

    replay(context);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    AvroDatumConverter<K, ?> keyConverter = converterFactory.create(
        (Class<K>) context.getOutputKeyClass());
    AvroDatumConverter<V, ?> valueConverter = converterFactory.create(
        (Class<V>) context.getOutputValueClass());

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
        dataModel, getCompressionCodec(context), getAvroFileOutputStream(context),
        getSyncInterval(context));
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
    if (null == writerSchema) {
      throw new IOException(
          "AvroKeyOutputFormat requires an output schema. Use AvroJob.setOutputKeySchema().");
    }

    GenericData dataModel = AvroSerialization.createDataModel(conf);

    return mRecordWriterFactory.create
      (writerSchema, dataModel, getCompressionCodec(context),
       getAvroFileOutputStream(context), getSyncInterval(context));
  }
View Full Code Here
TOP

Related Classes of org.apache.avro.generic.GenericData

  • org.apache.avro.hadoop.io.AvroSerialization
  • org.apache.avro.io.BinaryDecoder
  • org.apache.avro.io.BinaryEncoder
  • org.apache.avro.mapred.AvroJob
  • org.apache.avro.mapred.AvroOutputFormat
  • org.apache.avro.mapred.AvroSerialization
  • org.apache.avro.mapreduce.AvroKeyOutputFormat
  • org.apache.avro.mapreduce.AvroKeyValueOutputFormat
  • org.apache.avro.mapreduce.AvroRecordReaderBase
  • org.apache.avro.mapreduce.TestAvroKeyRecordWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.