Package org.apache.avro.generic.GenericData

Examples of org.apache.avro.generic.GenericData.Record

50
51
52
53
54
55
56
57
58
59
60
    try {
      writer = users.newWriter();
      Random rand = new Random();
      GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
      for (int i = 0; i < 100; i++) {
        Record record = builder.set("username", "user-" + i)
            .set("creationDate", System.currentTimeMillis())
            .set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
        writer.write(record);
      }
    } finally {
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    Path outPath = new Path(output);
    TupleFile.Reader reader = new TupleFile.Reader(FileSystem.get(outPath.toUri(), conf), conf, outPath);
    Tuple tuple = new Tuple(reader.getSchema());

    while(reader.next(tuple)) {
      Record record = (Record)tuple.get("my_avro");
      int topicId = (Integer) record.get("topic");
      String word = (record.get("word")).toString();
      int count   = (Integer) record.get("count");
      if(topicId == 1) {
        if(word.equals("bar") || word.equals("foo")) {
          assertEquals(2, count);
          validatedOutputLines++;
        } else if(word.equals("blah") || word.equals("bloh")) {
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
      throws NumberFormatException, IOException, InterruptedException {

    DatumReader datumReader = new SpecificDatumReader(AvroTweetsJoin.getAvroOutputSchema());
    FileReader reader = DataFileReader.openReader(new File(output),datumReader);
   
    Record record=null;
    record = (Record)reader.next(record);
   
    Assert.assertEquals("eric",record.get("username").toString());
    Array<Utf8> hashtags = (Array<Utf8>)record.get("hashtags");
    assertEquals(hashtags,"ivan","datasalt","pere");
   
    reader.next(record);
    Assert.assertEquals("eric",record.get("username").toString());
    assertEquals(hashtags,"ivan2","datasalt2","pere2");
    reader.next(record);
    Assert.assertEquals("marianico",record.get("username").toString());
    assertEquals(hashtags,"ivan2","datasalt2","pere2");
   
    Assert.assertFalse(reader.hasNext());
   
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    writer.setCodec(factory);
    int SYNC_SIZE = 16;
    int DEFAULT_SYNC_INTERVAL = 1000*SYNC_SIZE;
    writer.setSyncInterval(DEFAULT_SYNC_INTERVAL);
    writer.create(schema,new File(where));
    Record record = new Record(schema);
    record.put("id",1);
    record.put("text","1");
    record.put("timestamp",1L);
    record.put("hashtags",new String[]{"ivan","datasalt","pere"});
    writer.append(record);
   
    record.put("id",2);
    record.put("text","2");
    record.put("timestamp",2L);
    record.put("hashtags",new String[]{"ivan2","datasalt2","pere2"});
    writer.append(record);

    writer.close();
  }
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
      A a = (A) instance;
      a.setId(isRandom ? random.nextInt() + "" : "");
      a.setUrl(isRandom ? random.nextLong() + "" : "");
    } else if (field.getObjectSerialization() == AvroFieldSerialization.class) {
      if (instance == null || !(instance instanceof Record)) {
        instance = new Record(AVRO_SCHEMA);
      }
      Record record = (Record) instance;
      record.put("my_int", isRandom ? random.nextInt() : 0);
      record.put("my_string", isRandom ? random.nextLong() + "" : "");
    } else {
      throw new PangoolRuntimeException("Unknown field to fill");
    }
    tuple.set(index, instance);
  }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("tweet"));
    };
   
    public void map(AvroWrapper<Record> key, NullWritable value, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      Record tweet = key.datum();
      tuple.set("tweet_id",tweet.get("id"));
      tuple.set("tweet_hashtags",tweet.get("hashtags"));
      collector.write(tuple);
    }
View Full Code Here
103
104
105
106
107
108
109
110
    private Record outputRecord;
    private AvroWrapper<Record> wrapper;
   
    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      outputRecord= new Record(getAvroOutputSchema());
      wrapper = new AvroWrapper<Record>();
    };
View Full Code Here
76
77
78
79
80
81
82
83
    public void setup(TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      this.mapper = new ObjectMapper();
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema(0));
      record = new Record(getAvroSchema());
      tuple.set("my_avro",record);
    };
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    public void reduce(ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      int count = 0;
      ITuple outputTuple = null;
      Record outputRecord=null;
      for(ITuple tuple : tuples) {
        Record record = (Record)tuple.get("my_avro");
        count += (Integer) record.get("count");
        outputTuple = tuple;
        outputRecord = record;
      }
      outputRecord.put("count",count);
      outputTuple.set("my_avro",outputRecord);
View Full Code Here
65
66
67
68
69
70
71
72
73
    private final HadoopSerialization ser;
    private final DataOutputBuffer tmpOutputBuffer = new DataOutputBuffer();

    public TupleRecordWriter(org.apache.avro.Schema schema, Schema pangoolSchema,
        DataFileWriter<Record> writer, HadoopSerialization ser) {
      record = new Record(schema);
      this.ser = ser;
      this.writer = writer;
    }
View Full Code Here
TOP

Related Classes of org.apache.avro.generic.GenericData.Record

  • com.cloudera.cdk.data.filesystem.TestPartitionedDatasetWriter
  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$CountReducer
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$Red
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.