Examples of IndexedRecord

Examples of org.apache.avro.generic.IndexedRecord

92
93
94
95
96
97
98
99
  }

  public static class AvroIndexedRecordPartitioner<K, V> extends Partitioner<AvroKey<K>, AvroValue<V>> {
    @Override
    public int getPartition(AvroKey<K> key, AvroValue<V> value, int numPartitions) {
      IndexedRecord record = (IndexedRecord) key.datum();
      return (Math.abs(record.get(0).hashCode()) & Integer.MAX_VALUE) % numPartitions;
    }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
      return jsonObject;
    }

    case RECORD: {
      final ObjectNode jsonObject = JSON_NODE_FACTORY.objectNode();
      final IndexedRecord record = (IndexedRecord) value;
      if (!record.getSchema().equals(schema)) {
        throw new IOException(String.format(
            "Avro schema specifies record type '%s' but got '%s'.",
            schema.getFullName(), record.getSchema().getFullName()));
      }
      for (Schema.Field field : schema.getFields()) {
        final Object fieldValue = record.get(field.pos());
        final JsonNode fieldNode = toJsonNode(fieldValue, field.schema());
        // Outputs the field only if its value differs from the field's default:
        if ((field.defaultValue() == null) || !fieldNode.equals(field.defaultValue())) {
          jsonObject.put(field.name(), fieldNode);
        }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

93
94
95
96
97
98
99
100
  }

  public static class AvroIndexedRecordPartitioner<K, V> extends Partitioner<AvroKey<K>, AvroValue<V>> {
    @Override
    public int getPartition(AvroKey<K> key, AvroValue<V> value, int numPartitions) {
      IndexedRecord record = (IndexedRecord) key.datum();
      return (Math.abs(record.get(0).hashCode()) & Integer.MAX_VALUE) % numPartitions;
    }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

36
37
38
39
40
41
42
43
44
45
46
    avroPartitioner = new AvroIndexedRecordPartitioner();
  }

  @Test
  public void testGetPartition() {
    IndexedRecord indexedRecord = new MockIndexedRecord(3);
    AvroKey<IndexedRecord> avroKey = new AvroKey<IndexedRecord>(indexedRecord);

    assertEquals(3, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 5));
    assertEquals(1, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 2));
  }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

45
46
47
48
49
50
51
52
53
54
55
    assertEquals(1, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 2));
  }

  @Test
  public void testGetPartition_NegativeHashValue() {
    IndexedRecord indexedRecord = new MockIndexedRecord(-3);
    AvroKey<IndexedRecord> avroKey = new AvroKey<IndexedRecord>(indexedRecord);

    assertEquals(3, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 5));
    assertEquals(1, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 2));
  }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

54
55
56
57
58
59
60
61
62
63
    assertEquals(1, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), 2));
  }

  @Test
  public void testGetPartition_IntegerMinValue() {
    IndexedRecord indexedRecord = new MockIndexedRecord(Integer.MIN_VALUE);
    AvroKey<IndexedRecord> avroKey = new AvroKey<IndexedRecord>(indexedRecord);

    assertEquals(0, avroPartitioner.getPartition(avroKey, new AvroValue<Object>(), Integer.MAX_VALUE));
  }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

165
166
167
168
169
170
171
172
173
174
175
      AvroParquetReader<IndexedRecord> reader = null;
      try {
        reader = new AvroParquetReader(conf, path);
        while (true) {
          IndexedRecord datum;
          try {
            datum = reader.read();
          } catch (EOFException e) {
            return true; // be lenient
          }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

93
94
95
96
97
98
99
100
101
102
103
104
      if (avroResult == AvroConversions.ERROR) {
        LOG.debug("Cannot convert record: {} to schema: {}", inputRecord, schema);
        return false;         
      }
     
      IndexedRecord avroRecord = new GenericData.Record(schema);
      avroRecord.put(field.pos(), avroResult);
      outputRecord.put(Fields.ATTACHMENT_BODY, avroRecord);
     
      // pass record to next command in chain:
      return super.doProcess(outputRecord);
    }
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

291
292
293
294
295
296
297
298
299
300
301
302
303
      // RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES, INT, LONG, FLOAT,
      // DOUBLE, BOOLEAN, NULL
      switch (schema.getType()) {
      case RECORD: {
        IndexedRecord avroRecord = (IndexedRecord) datum;
        for (Field field : schema.getFields()) {
          flatten(avroRecord.get(field.pos()), field.schema(), list);
        }
        break;
      }
      case ENUM: {
        GenericEnumSymbol symbol = (GenericEnumSymbol) datum;
View Full Code Here

Examples of org.apache.avro.generic.IndexedRecord

97
98
99
100
101
102
103
104
105
106
107
108
109
110
    private void extractTree(Object datum, Schema schema, Record outputRecord, String prefix) {
      // RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES, INT, LONG, FLOAT,
      // DOUBLE, BOOLEAN, NULL
      switch (schema.getType()) {
      case RECORD: {
        IndexedRecord avroRecord = (IndexedRecord) datum;
        String prefix2 = prefix + "/";
        for (Field field : schema.getFields()) {
          extractTree(avroRecord.get(field.pos()), field.schema(), outputRecord, prefix2 + field.name());
        }
        break;
      }
      case ENUM: {
        GenericEnumSymbol symbol = (GenericEnumSymbol) datum;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.