Package org.apache.avro.generic.GenericData

Examples of org.apache.avro.generic.GenericData.Record

152
153
154
155
156
157
158
159
160
161
162
    super.stop();
    LOGGER.info("Twitter source {} stopped.", getName());
  }

  public void onStatus(Status status)  {
    Record doc = extractRecord("", avroSchema, status);
    if (doc == null) {
      return; // skip
    }
    docs.add(doc);
    if (docs.size() >= maxBatchSize ||
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
    return avroSchema;
  }

  private Record extractRecord(String idPrefix, Schema avroSchema, Status status) {
    User user = status.getUser();
    Record doc = new Record(avroSchema);

    doc.put("id", idPrefix + status.getId());
    doc.put("created_at", formatterTo.format(status.getCreatedAt()));
    doc.put("retweet_count", status.getRetweetCount());
    doc.put("retweeted", status.isRetweet());
    doc.put("in_reply_to_user_id", status.getInReplyToUserId());
    doc.put("in_reply_to_status_id", status.getInReplyToStatusId());

    addString(doc, "source", status.getSource());
    addString(doc, "text", status.getText());

    MediaEntity[] mediaEntities = status.getMediaEntities();
    if (mediaEntities.length > 0) {
      addString(doc, "media_url_https", mediaEntities[0].getMediaURLHttps());
      addString(doc, "expanded_url", mediaEntities[0].getExpandedURL());
    }

    doc.put("user_friends_count", user.getFriendsCount());
    doc.put("user_statuses_count", user.getStatusesCount());
    doc.put("user_followers_count", user.getFollowersCount());
    addString(doc, "user_location", user.getLocation());
    addString(doc, "user_description", user.getDescription());
    addString(doc, "user_screen_name", user.getScreenName());
    addString(doc, "user_name", user.getName());
    return doc;
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
                            + "\"fields\": ["
                            + "{\"name\": \"name\", \"type\": \"string\", \"order\": \"ascending\"}"
                            + "]}";

        AvroGenericSerializer serializer = new AvroGenericSerializer(jsonSchema);
        Record record = new Record(Schema.parse(jsonSchema));
        // we need to use a Utf8 instance to map to a String.
        record.put("name", new Utf8("Hello"));
        byte[] bytes = serializer.toBytes(record);
        assertTrue(serializer.toObject(bytes).equals(record));
    }
View Full Code Here
85
86
87
88
89
90
91
    private AvroKey<GenericData.Record> result ;
   
    @Override
    protected void setup(Context context) {
      result = new AvroKey<GenericData.Record>();
      result.datum(new Record(Pair.getPairSchema(STRING,LONG)));
    }
View Full Code Here
68
69
70
71
72
73
74
  }

  @Test(expected=AvroRuntimeException.class)
  public void testRecordCreateEmptySchema() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    Record r = new GenericData.Record(s);
  }
View Full Code Here
89
90
91
92
93
94
95
96
  public void testRecordPutInvalidField() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    List<Schema.Field> fields = new ArrayList<Schema.Field>();
    fields.add(new Schema.Field("someFieldName", s, "docs", null));
    s.setFields(fields);
    Record r = new GenericData.Record(s);
    r.put("invalidFieldName", "someValue");
  }
View Full Code Here
345
346
347
348
349
350
351
352
353
354
355
356
357
358
    ByteArrayOutputStream b1 = new ByteArrayOutputStream(5);
    ByteArrayOutputStream b2 = new ByteArrayOutputStream(5);
    BinaryEncoder b1Enc = EncoderFactory.get().binaryEncoder(b1, null);
    BinaryEncoder b2Enc = EncoderFactory.get().binaryEncoder(b2, null);
    // Prepare two different datums
    Record testDatum1 = new Record(record);
    testDatum1.put(0, 1);
    Record testDatum2 = new Record(record);
    testDatum2.put(0, 2);
    GenericDatumWriter<Record> gWriter = new GenericDatumWriter<Record>(record);
    Integer start1 = 0, start2 = 0;
    try {
      // Write two datums in each stream
      // and get the offset length after the first write in each.
View Full Code Here
68
69
70
71
72
73
74
  }

  @Test(expected=AvroRuntimeException.class)
  public void testRecordCreateEmptySchema() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    Record r = new GenericData.Record(s);
  }
View Full Code Here
89
90
91
92
93
94
95
96
  public void testRecordPutInvalidField() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    List<Schema.Field> fields = new ArrayList<Schema.Field>();
    fields.add(new Schema.Field("someFieldName", s, "docs", null));
    s.setFields(fields);
    Record r = new GenericData.Record(s);
    r.put("invalidFieldName", "someValue");
  }
View Full Code Here
330
331
332
333
334
335
336
337
338
339
340
341
342
343
    ByteArrayOutputStream b1 = new ByteArrayOutputStream(5);
    ByteArrayOutputStream b2 = new ByteArrayOutputStream(5);
    BinaryEncoder b1Enc = EncoderFactory.get().binaryEncoder(b1, null);
    BinaryEncoder b2Enc = EncoderFactory.get().binaryEncoder(b2, null);
    // Prepare two different datums
    Record testDatum1 = new Record(record);
    testDatum1.put(0, 1);
    Record testDatum2 = new Record(record);
    testDatum2.put(0, 2);
    GenericDatumWriter<Record> gWriter = new GenericDatumWriter<Record>(record);
    Integer start1 = 0, start2 = 0;
    try {
      // Write two datums in each stream
      // and get the offset length after the first write in each.
View Full Code Here
TOP

Related Classes of org.apache.avro.generic.GenericData.Record

  • com.cloudera.cdk.data.filesystem.TestPartitionedDatasetWriter
  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$CountReducer
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$Red
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.