Package org.apache.avro.generic

Examples of org.apache.avro.generic.GenericRecordBuilder

130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
    this.in = new MemoryChannel();
    Configurables.configure(in, config);

    config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI, FILE_DATASET_URI);

    GenericRecordBuilder builder = new GenericRecordBuilder(RECORD_SCHEMA);
    expected = Lists.<GenericRecord>newArrayList(
        builder.set("id", "1").set("msg", "msg1").build(),
        builder.set("id", "2").set("msg", "msg2").build(),
        builder.set("id", "3").set("msg", "msg3").build());

    putToChannel(in, Iterables.transform(expected,
        new Function<GenericRecord, Event>() {
          private int i = 0;
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
  }

  @Test
  public void testDatasetUpdate() throws EventDeliveryException {
    // add an updated record that is missing the msg field
    GenericRecordBuilder updatedBuilder = new GenericRecordBuilder(
      UPDATED_SCHEMA);
    GenericData.Record updatedRecord = updatedBuilder
      .set("id", "0")
      .set("priority", 1)
      .set("msg", "Priority 1 message!")
      .build();

    // make a set of the expected records with the new schema
    Set<GenericRecord> expectedAsUpdated = Sets.newHashSet();
    for (GenericRecord record : expected) {
      expectedAsUpdated.add(updatedBuilder
        .clear("priority")
        .set("id", record.get("id"))
        .set("msg", record.get("msg"))
        .build());
    }
View Full Code Here
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
  @Test
  public void testCompatibleSchemas() throws EventDeliveryException {
    DatasetSink sink = sink(in, config);

    // add a compatible record that is missing the msg field
    GenericRecordBuilder compatBuilder = new GenericRecordBuilder(
        COMPATIBLE_SCHEMA);
    GenericData.Record compatibleRecord = compatBuilder.set("id", "0").build();

    // add the record to the incoming channel
    putToChannel(in, event(compatibleRecord, COMPATIBLE_SCHEMA, null, false));

    // the record will be read using the real schema, so create the expected
    // record using it, but without any data

    GenericRecordBuilder builder = new GenericRecordBuilder(RECORD_SCHEMA);
    GenericData.Record expectedRecord = builder.set("id", "0").build();
    expected.add(expectedRecord);

    // run the sink
    sink.start();
    sink.process();
View Full Code Here
461
462
463
464
465
466
467
468
469
470
471
472
473
  @Test
  public void testIncompatibleSchemas() throws EventDeliveryException {
    final DatasetSink sink = sink(in, config);

    GenericRecordBuilder builder = new GenericRecordBuilder(
        INCOMPATIBLE_SCHEMA);
    GenericData.Record rec = builder.set("username", "koala").build();
    putToChannel(in, event(rec, INCOMPATIBLE_SCHEMA, null, false));

    // run the sink
    sink.start();
    assertThrows("Should fail", EventDeliveryException.class,
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
                .name("name").type().stringType().noDefault()
                .name("ID").type().intType().noDefault()
                .endRecord();


        user = new GenericRecordBuilder(schema)
                .set("name", "Jeff")
                .set("ID", 1)
                .build();

View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
        .record("MyRecord").namespace("ns")
        .fields()
          .name("field1").type().stringType().noDefault()
          .name("field2").type().stringType().noDefault()
        .endRecord();
    GenericData.Record record = new GenericRecordBuilder(schemaWithoutField).set("field1", "someValue").build();
    byte[] data = writeRecord(schemaWithoutField, record);
    try {
      readRecord(schemaWithField, data);
      Assert.fail("Expected exception not thrown");
    } catch (AvroTypeException typeException) {
View Full Code Here
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
    Schema rec2 = SchemaBuilder.record("inner2").fields()
      .name("f2").type().intType().noDefault()
      .endRecord();
   
    GenericData.Record recdef =
        new GenericRecordBuilder(rec).set("f", 1).build();
       
    GenericData.Record recdef2 =
        new GenericRecordBuilder(rec2).set("f2", 2).build();
   
    Schema r = SchemaBuilder.record("r").fields()
      .name("boolF").type().booleanType().booleanDefault(false)
      .name("intF").type().intType().intDefault(1)
      .name("longF").type().longType().longDefault(2L)
      .name("floatF").type().floatType().floatDefault(3.0f)
      .name("doubleF").type().doubleType().doubleDefault(4.0d)
      .name("stringF").type().stringType().stringDefault("def")
      .name("bytesF1").type().bytesType().bytesDefault(bytedef)
      .name("bytesF2").type().bytesType().bytesDefault(bufdef)
      .name("bytesF3").type().bytesType().bytesDefault(strdef)
      .name("nullF").type().nullType().nullDefault()
      .name("fixedF1").type().fixed("F1").size(1).fixedDefault(bytedef)
      .name("fixedF2").type().fixed("F2").size(1).fixedDefault(bufdef)
      .name("fixedF3").type().fixed("F3").size(1).fixedDefault(strdef)
      .name("enumF").type().enumeration("E1").symbols("S").enumDefault("S")
      .name("mapF").type().map().values().stringType()
        .mapDefault(mapdef)
      .name("arrayF").type().array().items().stringType()
        .arrayDefault(arrdef)
      .name("recordF").type().record("inner").fields()
        .name("f").type().intType().noDefault()
        .endRecord().recordDefault(recdef)
      .name("byName").type("E1").withDefault("S")
      // union builders, one for each 'first type' in a union:
      .name("boolU").type().unionOf().booleanType().and()
        .intType().endUnion().booleanDefault(false)
      .name("intU").type().unionOf().intType().and()
        .longType().endUnion().intDefault(1)
      .name("longU").type().unionOf().longType().and()
        .intType().endUnion().longDefault(2L)
      .name("floatU").type().unionOf().floatType().and()
        .intType().endUnion().floatDefault(3.0f)
      .name("doubleU").type().unionOf().doubleType().and()
        .intType().endUnion().doubleDefault(4.0d)
      .name("stringU").type().unionOf().stringType().and()
        .intType().endUnion().stringDefault("def")
      .name("bytesU").type().unionOf().bytesType().and()
        .intType().endUnion().bytesDefault(bytedef)
      .name("nullU").type().unionOf().nullType().and()
        .intType().endUnion().nullDefault()
      .name("fixedU").type().unionOf().fixed("F4").size(1).and()
        .intType().endUnion().fixedDefault(bytedef)
      .name("enumU").type().unionOf().enumeration("E2").symbols("SS").and()
        .intType().endUnion().enumDefault("SS")
      .name("mapU").type().unionOf().map().values().stringType().and()
        .intType().endUnion().mapDefault(mapdef)
      .name("arrayU").type().unionOf().array().items().stringType().and()
        .intType().endUnion().arrayDefault(arrdef)
      .name("recordU").type().unionOf().record("inner2").fields()
        .name("f2").type().intType().noDefault()
        .endRecord().and().intType().endUnion().recordDefault(recdef2)
      .endRecord();
   
    GenericData.Record newRec =
        new GenericRecordBuilder(r).build();
   
    Assert.assertEquals(false, newRec.get("boolF"));
    Assert.assertEquals(false, newRec.get("boolU"));
    Assert.assertEquals(1, newRec.get("intF"));
    Assert.assertEquals(1, newRec.get("intU"));
View Full Code Here
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
      .name("requiredInt").type().intType().noDefault()
      .name("optionalInt").type().optional().intType()
      .name("nullableIntWithDefault").type().nullable().intType().intDefault(3)
      .endRecord();

    GenericData.Record rec1 = new GenericRecordBuilder(writeSchema)
        .set("requiredInt", 1)
        .build();

    Assert.assertEquals(1, rec1.get("requiredInt"));
    Assert.assertEquals(null, rec1.get("optionalInt"));
    Assert.assertEquals(3, rec1.get("nullableIntWithDefault"));

    GenericData.Record rec2 = new GenericRecordBuilder(writeSchema)
        .set("requiredInt", 1)
        .set("optionalInt", 2)
        .set("nullableIntWithDefault", 13)
        .build();
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
          throw new StoreException("Invalid format " + datasetDefinition.getFormat() +
              " specified, you must use 'parquet' with " + this.getClass().getSimpleName() + ".");
        }
      }
    }
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    BeanWrapper beanWrapper = PropertyAccessorFactory.forBeanPropertyAccess(entity);
    for (Schema.Field f : schema.getFields()) {
      if (beanWrapper.isReadableProperty(f.name())) {
        Schema fieldSchema = f.schema();
        if (f.schema().getType().equals(Schema.Type.UNION)) {
          for (Schema s : f.schema().getTypes()) {
            if (!s.getName().equals("null")) {
              fieldSchema = s;
            }
          }
        }
        if (fieldSchema.getType().equals(Schema.Type.RECORD)) {
          throw new StoreException("Nested record currently not supported for field: " + f.name() +
              " of type: " + beanWrapper.getPropertyDescriptor(f.name()).getPropertyType().getName());
        } else {
          if (fieldSchema.getType().equals(Schema.Type.BYTES)) {
            ByteBuffer buffer = null;
            Object value = beanWrapper.getPropertyValue(f.name());
            if (value == null || value instanceof byte[]) {
              if(value != null) {
                byte[] bytes = (byte[]) value;
                buffer = ByteBuffer.wrap(bytes);
              }
              builder.set(f.name(), buffer);
            } else {
              throw new StoreException("Don't know how to handle " + value.getClass() + " for " + fieldSchema);
            }
          } else {
            builder.set(f.name(), beanWrapper.getPropertyValue(f.name()));
            }
        }
      }
    }
    try {
      writer.write(builder.build());
    } catch (ClassCastException cce) {
      throw new StoreException("Failed to write record with schema: " +
          schema, cce);
    }
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_NAME, DATASET_NAME);

    this.in = new MemoryChannel();
    Configurables.configure(in, config);

    GenericRecordBuilder builder = new GenericRecordBuilder(RECORD_SCHEMA);
    expected = Lists.newArrayList(
        builder.set("id", "1").set("msg", "msg1").build(),
        builder.set("id", "2").set("msg", "msg2").build(),
        builder.set("id", "3").set("msg", "msg3").build());

    putToChannel(in, Iterables.transform(expected,
        new Function<GenericData.Record, Event>() {
          private int i = 0;
View Full Code Here
TOP

Related Classes of org.apache.avro.generic.GenericRecordBuilder

  • com.cloudera.cdk.data.filesystem.DatasetTestUtilities
  • com.cloudera.cdk.data.filesystem.TestPartitionedDatasetWriter
  • com.cloudera.cdk.examples.data.CreateHCatalogUserDatasetGeneric
  • com.cloudera.cdk.examples.data.CreateUserDatasetGeneric
  • com.cloudera.cdk.examples.data.CreateUserDatasetGenericParquet
  • com.cloudera.cdk.examples.data.CreateUserDatasetGenericPartitioned
  • com.cloudera.cdk.examples.logging.App
  • com.cloudera.cdk.examples.logging.webapp.LoggingServlet
  • com.cloudera.cdk.examples.staging.GenerateSimpleLogs
  • com.cloudera.cdk.examples.staging.StagingToPersistentSerial
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.