Examples of org.apache.avro.Schema.Field

org.apache.avro.Schema.Field

import org.apache.avro.file.DataFileWriter;


public class TestSchema {
  public static Schema schemaFromCode() {
    List<Field> fields = new ArrayList<Field>();
    fields.add(new Field("field1", Schema.create(Type.STRING), null, null));
    fields.add(new Field("field2", Schema.create(Type.LONG), null, null));
    fields.add(new Field("field3", Schema.createArray(Schema.create(Type.STRING)), null, null));
    fields.add(new Field("field4", Schema.createMap(Schema.create(Type.INT)), null, null));


    Schema schema = Schema.createRecord("recordName", "Record Doc String", "recordNS", false);
    schema.setFields(fields);


    schema.addProp("Property1", "Value1");

View Full Code Here

      Delete delete = new Delete(keyRaw);
      boolean hasPuts = false;
      boolean hasDeletes = false;
      Iterator<Field> iter = schema.getFields().iterator();
      for (int i = 0; iter.hasNext(); i++) {
        Field field = iter.next();
        if (!stateManager.isDirty(persistent, i)) {
          continue;
        }
        Type type = field.schema().getType();
        Object o = persistent.get(i);
        HBaseColumn hcol = mapping.getColumn(field.name());
        if (hcol == null) {
          throw new RuntimeException("HBase mapping for field ["+ persistent.getClass().getName() +
              "#"+ field.name()+"] not found. Wrong gora-hbase-mapping.xml?");
        }
        switch(type) {
          case MAP:
            if(o instanceof StatefulMap) {
              StatefulHashMap<Utf8, ?> map = (StatefulHashMap<Utf8, ?>) o;
              for (Entry<Utf8, State> e : map.states().entrySet()) {
                Utf8 mapKey = e.getKey();
                switch (e.getValue()) {
                  case DIRTY:
                    byte[] qual = Bytes.toBytes(mapKey.toString());
                    byte[] val = toBytes(map.get(mapKey), field.schema().getValueType());
                    put.add(hcol.getFamily(), qual, val);
                    hasPuts = true;
                    break;
                  case DELETED:
                    qual = Bytes.toBytes(mapKey.toString());
                    hasDeletes = true;
                    delete.deleteColumn(hcol.getFamily(), qual);
                    break;
                }
              }
            } else {
              Set<Map.Entry> set = ((Map)o).entrySet();
              for(Entry entry: set) {
                byte[] qual = toBytes(entry.getKey());
                byte[] val = toBytes(entry.getValue());
                put.add(hcol.getFamily(), qual, val);
                hasPuts = true;
              }
            }
            break;
          case ARRAY:
            if(o instanceof GenericArray) {
              GenericArray arr = (GenericArray) o;
              int j=0;
              for(Object item : arr) {
                byte[] val = toBytes(item);
                put.add(hcol.getFamily(), Bytes.toBytes(j++), val);
                hasPuts = true;
              }
            }
            break;
          default:
            put.add(hcol.getFamily(), hcol.getQualifier(), toBytes(o, field.schema()));
            hasPuts = true;
            break;
        }
      }
      if (hasPuts) {

View Full Code Here

      HBaseColumn col = mapping.getColumn(f);
      if (col == null) {
        throw new  RuntimeException("HBase mapping for field ["+ f +"] not found. " +
            "Wrong gora-hbase-mapping.xml?");
      }
      Field field = fieldMap.get(f);
      Schema fieldSchema = field.schema();
      switch(fieldSchema.getType()) {
        case MAP:
          NavigableMap<byte[], byte[]> qualMap =
            result.getNoVersionMap().get(col.getFamily());
          if (qualMap == null) {

View Full Code Here

    Map currentMap = null;
    ArrayList currentArray = null;
    Text currentFam = null;
    int currentPos = 0;
    Schema currentSchema = null;
    Field currentField = null;


    while (iter.hasNext()) {
      Entry<Key,Value> entry = iter.next();
      
      if (currentMap != null) {
        if (currentFam.equals(entry.getKey().getColumnFamily())) {
          currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), fromBytes(currentSchema, entry.getValue().get()));
          continue;
        } else {
          persistent.put(currentPos, currentMap);
          currentMap = null;
        }
      } else if (currentArray != null) {
        if (currentFam.equals(entry.getKey().getColumnFamily())) {
          currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
          continue;
        } else {
          persistent.put(currentPos, new ListGenericArray<T>(currentField.schema(), currentArray));
          currentArray = null;
        }
      }


      if (row == null)
        row = entry.getKey().getRowData();
      
      String fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()));
      if (fieldName == null)
        fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), null));


      Field field = fieldMap.get(fieldName);


      switch (field.schema().getType()) {
        case MAP:
          currentMap = new StatefulHashMap();
          currentPos = field.pos();
          currentFam = entry.getKey().getColumnFamily();
          currentSchema = field.schema().getValueType();
          
          currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()), fromBytes(currentSchema, entry.getValue().get()));


          break;
        case ARRAY:
          currentArray = new ArrayList();
          currentPos = field.pos();
          currentFam = entry.getKey().getColumnFamily();
          currentSchema = field.schema().getElementType();
          currentField = field;
          
          currentArray.add(fromBytes(currentSchema, entry.getValue().get()));


          break;
        case RECORD:
          SpecificDatumReader reader = new SpecificDatumReader(field.schema());
          byte[] val = entry.getValue().get();
          // TODO reuse decoder
          BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(val, null);
          persistent.put(field.pos(), reader.read(null, decoder));
          break;
        default:
          persistent.put(field.pos(), fromBytes(field.schema(), entry.getValue().get()));
      }
    }
    
    if (currentMap != null) {
      persistent.put(currentPos, currentMap);

View Full Code Here

      String family = cassandraColumn.getFamily();
      String fieldName = this.reverseMap.get(family + ":" + StringSerializer.get().fromByteBuffer(cassandraColumn.getName()));
      
      // get field
      int pos = this.persistent.getFieldIndex(fieldName);
      Field field = fields.get(pos);
      
      // get value
      cassandraColumn.setField(field);
      Object value = cassandraColumn.getValue();

View Full Code Here

      
      Iterator<Field> iter = schema.getFields().iterator();
      
      int count = 0;
      for (int i = 0; iter.hasNext(); i++) {
        Field field = iter.next();
        if (!stateManager.isDirty(val, i)) {
          continue;
        }
        
        Object o = val.get(i);
        Pair<Text,Text> col = mapping.fieldMap.get(field.name());
  
        switch (field.schema().getType()) {
          case MAP:
            if (o instanceof StatefulMap) {
              StatefulMap map = (StatefulMap) o;
              Set<?> es = map.states().entrySet();
              for (Object entry : es) {
                Object mapKey = ((Entry) entry).getKey();
                State state = (State) ((Entry) entry).getValue();
  
                switch (state) {
                  case NEW:
                  case DIRTY:
                    m.put(col.getFirst(), new Text(toBytes(mapKey)), new Value(toBytes(map.get(mapKey))));
                    count++;
                    break;
                  case DELETED:
                    m.putDelete(col.getFirst(), new Text(toBytes(mapKey)));
                    count++;
                    break;
                }
                
              }
            } else {
              Map map = (Map) o;
              Set<?> es = map.entrySet();
              for (Object entry : es) {
                Object mapKey = ((Entry) entry).getKey();
                Object mapVal = ((Entry) entry).getValue();
                m.put(col.getFirst(), new Text(toBytes(mapKey)), new Value(toBytes(mapVal)));
                count++;
              }
            }
            break;
          case ARRAY:
            GenericArray array = (GenericArray) o;
            int j = 0;
            for (Object item : array) {
              m.put(col.getFirst(), new Text(toBytes(j++)), new Value(toBytes(item)));
              count++;
            }
            break;
          case RECORD:
            SpecificDatumWriter writer = new SpecificDatumWriter(field.schema());
            ByteArrayOutputStream os = new ByteArrayOutputStream();
            BinaryEncoder encoder = new BinaryEncoder(os);
            writer.write(o, encoder);
            encoder.flush();
            m.put(col.getFirst(), col.getSecond(), new Value(os.toByteArray()));

View Full Code Here

    List<Field> fields = new ArrayList<Field>();
    for (String columnName : columnNames) {
      String cleanedCol = ClassWriter.toIdentifier(columnName);
      int sqlType = columnTypes.get(columnName);
      Schema avroSchema = toAvroSchema(sqlType, columnName);
      Field field = new Field(cleanedCol, avroSchema, null, null);
      field.addProp("columnName", columnName);
      field.addProp("sqlType", Integer.toString(sqlType));
      fields.add(field);
    }


    TableClassName tableClassName = new TableClassName(options);
    String shortClassName = tableClassName.getShortClassForTable(tableName);

View Full Code Here

        }


        // Check all of the fields in the new schema
        for(Field newField: newSchema.getFields()) {
            String fieldName = newField.name();
            Field oldField = oldSchema.getField(fieldName);


            if(oldField == null) {
                // This is a new field that did not exist in the original
                // schema.
                // Check if it is optional or has a default value.
                if(isOptional(newField)) {
                    if(newField.defaultValue() == null) {
                        messages.add(new Message(Level.INFO, "Added optional field " + name + "."
                                                             + fieldName
                                                             + " with no default value."));
                    } else {
                        messages.add(new Message(Level.INFO, "Added optional field " + name + "."
                                                             + fieldName + " with default value: "
                                                             + newField.defaultValue()));
                    }
                } else {
                    if(newField.defaultValue() == null) {
                        messages.add(new Message(Level.ERROR, "Added required field " + name + "."
                                                              + fieldName
                                                              + " with no default value."));
                    } else {
                        messages.add(new Message(Level.INFO, "Added required field " + name + "."
                                                             + fieldName + " with default value: "
                                                             + newField.defaultValue()));
                    }
                }
            } else {
                // This is a field that existed in the original schema.


                // Check if the field was changed from optional to required or
                // vice versa.
                boolean newFieldIsOptional = isOptional(newField);
                boolean oldFieldIsOptional = isOptional(oldField);


                if(oldFieldIsOptional != newFieldIsOptional) {
                    if(oldFieldIsOptional) {
                        messages.add(new Message(Level.ERROR,
                                                 "Existing field " + name + "." + fieldName
                                                         + " was optional and is now required."));
                    } else {
                        messages.add(new Message(Level.WARN, "Existing field " + name + "."
                                                             + fieldName
                                                             + " was required and is now optional."));
                    }
                }


                // Recursively compare the nested field types
                compareTypes(oldField.schema(), newField.schema(), messages, name + "." + fieldName);


                // Check if the default value has been changed
                if(newField.defaultValue() == null) {
                    if(oldField.defaultValue() != null) {
                        messages.add(new Message(Level.WARN,
                                                 "Removed default value for existing field " + name
                                                         + "." + fieldName
                                                         + ". The old default was: "
                                                         + oldField.defaultValue()));
                    }
                } else // newField.defaultValue() != null
                {
                    if(oldField.defaultValue() == null) {
                        messages.add(new Message(Level.WARN,
                                                 "Added a default value for existing field " + name
                                                         + "." + fieldName
                                                         + ". The new default is: "
                                                         + newField.defaultValue()));
                    } else if(!newField.defaultValue().equals(oldField.defaultValue())) {
                        messages.add(new Message(Level.INFO,
                                                 "Changed the default value for existing field "
                                                         + name + "." + fieldName
                                                         + ". The old default was: "
                                                         + oldField.defaultValue()
                                                         + ". The new default is: "
                                                         + newField.defaultValue()));
                    }
                }
            }


            // For all fields in the new schema (whether or not it existed in
            // the old schema), if there is a default value for this field, make
            // sure it is legal.
            if(newField.defaultValue() != null) {
                checkDefaultValueIsLegal(newField, messages, name + "." + fieldName);
            }
        }


        // Check if any fields were removed.
        for(Field oldField: newSchema.getFields()) {
            String fieldName = oldField.name();
            Field newField = newSchema.getField(fieldName);


            if(newField == null) {
                if(isOptional(oldField)) {
                    messages.add(new Message(Level.INFO, "Removed optional field " + name + "."
                                                         + fieldName));

View Full Code Here

        Schema schema = Schema.parse(vschema.getSchema().toString());
        LOG.info("Schema =" + vschema.getSchema() + "version="
            + vschema.getVersion() + " name=" + vschema.getSchemaBaseName());


        /* Determine type of field txn */
        Field txnFieldType = schema.getField("txn");
        if (txnFieldType == null)
        {
          throw new Exception(
              "Unable to find field called 'txn'. Cannot proceeed\n");
        }
        Type txnType = SchemaHelper.getAnyType(txnFieldType);


        /*
         * Determine primary key of schema. This is assumed to be invariant
         * across versions
         */
        String keyOverrideName = SchemaHelper.getMetaField(schema, "pk");
        String keyColumnName = "key";
        if (null != keyOverrideName)
        {
          keyColumnName = keyOverrideName;
        }
        Field pkeyField = schema.getField(keyColumnName);
        if (null == pkeyField)
        {
          keyColumnName = "id";
          pkeyField = schema.getField("id");
        }

View Full Code Here

      GenericRecord payload1 = avroDecoder.getGenericRecord(e, null);
      GenericRecord metadata1 = avroDecoder.getMetadata(e, null);


      Schema pschema = Schema.createUnion(Arrays.asList(avroDecoder.getPayloadSchema(e).getSchema(),
                                                        Schema.create(Type.NULL)));
      Field pfield = new Field("payload", pschema, "payload", null);
      VersionedSchema metaschema = avroDecoder.getMetadataSchema(e);
      Schema mschema = null != metaschema ?
          Schema.createUnion(Arrays.asList(metaschema.getSchema(), Schema.create(Type.NULL))) :
          Schema.createUnion(Arrays.asList(Schema.create(Type.INT), Schema.create(Type.NULL)));
      Field mfield = new Field("metadata", mschema, "metadata", null);
      Schema combined = Schema.createRecord(Arrays.asList(pfield, mfield));
      GenericRecord r = new GenericData.Record(combined);
      r.put(0, payload1);
      r.put(1, metadata1);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.avro.Schema.Field

co.nubetech.hiho.avro.DBMapper

co.nubetech.hiho.avro.TestDBMapper

com.cloudera.cdk.data.filesystem.TestFileSystemDatasetReader

com.cloudera.cdk.data.hbase.avro.AvroEntityComposer

com.cloudera.cdk.data.hbase.avro.AvroEntitySchema

com.cloudera.cdk.data.hbase.avro.AvroEntitySerDe

com.cloudera.cdk.data.hbase.avro.AvroUtils

com.cloudera.cdk.morphline.avro.AvroMorphlineTest

com.cloudera.sqoop.TestAvroExport

com.cloudera.sqoop.TestParquetExport

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.