Package org.apache.avro.Schema

Examples of org.apache.avro.Schema.Field


   * @return the writer field, if any does correspond, or None.
   */
  public static Field lookupWriterField(final Schema writerSchema, final Field readerField) {
    assert (writerSchema.getType() == Type.RECORD);
    final List<Field> writerFields = new ArrayList<Field>();
    final Field direct = writerSchema.getField(readerField.name());
    if (direct != null) {
      writerFields.add(direct);
    }
    for (final String readerFieldAliasName : readerField.aliases()) {
      final Field writerField = writerSchema.getField(readerFieldAliasName);
      if (writerField != null) {
        writerFields.add(writerField);
      }
    }
    switch (writerFields.size()) {
View Full Code Here


              return SchemaCompatibilityType.INCOMPATIBLE;
            }

            // Check that each field in the reader record can be populated from the writer record:
            for (final Field readerField : reader.getFields()) {
              final Field writerField = lookupWriterField(writer, readerField);
              if (writerField == null) {
                // Reader field does not correspond to any field in the writer record schema,
                // reader field must have a default value.
                if (readerField.defaultValue() == null) {
                  // reader field has no default value
                  return SchemaCompatibilityType.INCOMPATIBLE;
                }
              } else {
                if (getCompatibility(readerField.schema(), writerField.schema())
                    == SchemaCompatibilityType.INCOMPATIBLE) {
                  return SchemaCompatibilityType.INCOMPATIBLE;
                }
              }
            }
View Full Code Here

      Field[] reordered = new Field[rfields.size()];
      int ridx = 0;
      int count = 1 + wfields.size();

      for (Field f : wfields) {
        Field rdrField = reader.getField(f.name());
        if (rdrField != null) {
          reordered[ridx++] = rdrField;
        }
      }

      for (Field rf : rfields) {
        String fname = rf.name();
        if (writer.getField(fname) == null) {
          if (rf.defaultValue() == null) {
            result = Symbol.error("Found " + writer.getFullName()
                                  + ", expecting " + reader.getFullName()
                                  + ", missing required field " + fname);
            seen.put(wsc, result);
            return result;
          } else {
            reordered[ridx++] = rf;
            count += 3;
          }
        }
      }

      Symbol[] production = new Symbol[count];
      production[--count] = Symbol.fieldOrderAction(reordered);

      /**
       * We construct a symbol without filling the array. Please see
       * {@link Symbol#production} for the reason.
       */
      result = Symbol.seq(production);
      seen.put(wsc, result);

      /*
       * For now every field in read-record with no default value
       * must be in write-record.
       * Write record may have additional fields, which will be
       * skipped during read.
       */

      // Handle all the writer's fields
      for (Field wf : wfields) {
        String fname = wf.name();
        Field rf = reader.getField(fname);
        if (rf == null) {
          production[--count] =
            Symbol.skipAction(generate(wf.schema(), wf.schema(), seen));
        } else {
          production[--count] =
            generate(wf.schema(), rf.schema(), seen);
        }
      }

      // Add default values for fields missing from Writer
      for (Field rf : rfields) {
        String fname = rf.name();
        Field wf = writer.getField(fname);
        if (wf == null) {
          byte[] bb = getBinary(rf.schema(), rf.defaultValue());
          production[--count] = Symbol.defaultStartAction(bb);
          production[--count] = generate(rf.schema(), rf.schema(), seen);
          production[--count] = Symbol.DEFAULT_END_ACTION;
View Full Code Here

    @Override public void put(String key, Object value) {
      values[schema.getField(key).pos()] = value;
    }
    @Override public void put(int i, Object v) { values[i] = v; }
    @Override public Object get(String key) {
      Field field = schema.getField(key);
      if (field == null) return null;
      return values[field.pos()];
    }
View Full Code Here

    Delete delete = new Delete(keyRaw);
    boolean hasPuts = false;
    boolean hasDeletes = false;
    Iterator<Field> iter = schema.getFields().iterator();
    for (int i = 0; iter.hasNext(); i++) {
      Field field = iter.next();
      if (!stateManager.isDirty(persistent, i)) {
        continue;
      }
      Type type = field.schema().getType();
      Object o = persistent.get(i);
      HBaseColumn hcol = mapping.getColumn(field.name());
      switch(type) {
        case MAP:
          if(o instanceof StatefulMap) {
            StatefulHashMap<Utf8, ?> map = (StatefulHashMap<Utf8, ?>) o;
            for (Entry<Utf8, State> e : map.states().entrySet()) {
              Utf8 mapKey = e.getKey();
              switch (e.getValue()) {
                case DIRTY:
                  byte[] qual = Bytes.toBytes(mapKey.toString());
                  byte[] val = toBytes(map.get(mapKey), field.schema().getValueType());
                  put.add(hcol.getFamily(), qual, val);
                  hasPuts = true;
                  break;
                case DELETED:
                  qual = Bytes.toBytes(mapKey.toString());
                  hasDeletes = true;
                  delete.deleteColumn(hcol.getFamily(), qual);
                  break;
              }
            }
          } else {
            Set<Map.Entry> set = ((Map)o).entrySet();
            for(Entry entry: set) {
              byte[] qual = toBytes(entry.getKey());
              byte[] val = toBytes(entry.getValue());
              put.add(hcol.getFamily(), qual, val);
              hasPuts = true;
            }
          }
          break;
        case ARRAY:
          if(o instanceof GenericArray) {
            GenericArray arr = (GenericArray) o;
            int j=0;
            for(Object item : arr) {
              byte[] val = toBytes(item);
              put.add(hcol.getFamily(), Bytes.toBytes(j++), val);
              hasPuts = true;
            }
          }
          break;
        default:
          put.add(hcol.getFamily(), hcol.getQualifier(), toBytes(o, field.schema()));
          hasPuts = true;
          break;
      }
    }
    if (hasPuts) {
View Full Code Here

    T persistent = newPersistent();
    StateManager stateManager = persistent.getStateManager();
    for (String f : fields) {
      HBaseColumn col = mapping.getColumn(f);
      Field field = fieldMap.get(f);
      Schema fieldSchema = field.schema();
      switch(fieldSchema.getType()) {
        case MAP:
          NavigableMap<byte[], byte[]> qualMap =
            result.getNoVersionMap().get(col.getFamily());
          if (qualMap == null) {
View Full Code Here

      values[field.pos()] = value;
    }
    @Override public void put(int i, Object v) { values[i] = v; }
    @Override public Object get(String key) {
      Field field = schema.getField(key);
      if (field == null) return null;
      return values[field.pos()];
    }
View Full Code Here

      Field[] reordered = new Field[rfields.size()];
      int ridx = 0;
      int count = 1 + wfields.size();

      for (Field f : wfields) {
        Field rdrField = reader.getField(f.name());
        if (rdrField != null) {
          reordered[ridx++] = rdrField;
        }
      }

      for (Field rf : rfields) {
        String fname = rf.name();
        if (writer.getField(fname) == null) {
          if (rf.defaultValue() == null) {
            result = Symbol.error("Found " + writer.getFullName()
                                  + ", expecting " + reader.getFullName()
                                  + ", missing required field " + fname);
            seen.put(wsc, result);
            return result;
          } else {
            reordered[ridx++] = rf;
            count += 3;
          }
        }
      }

      Symbol[] production = new Symbol[count];
      production[--count] = Symbol.fieldOrderAction(reordered);

      /**
       * We construct a symbol without filling the array. Please see
       * {@link Symbol#production} for the reason.
       */
      result = Symbol.seq(production);
      seen.put(wsc, result);

      /*
       * For now every field in read-record with no default value
       * must be in write-record.
       * Write record may have additional fields, which will be
       * skipped during read.
       */

      // Handle all the writer's fields
      for (Field wf : wfields) {
        String fname = wf.name();
        Field rf = reader.getField(fname);
        if (rf == null) {
          production[--count] =
            Symbol.skipAction(generate(wf.schema(), wf.schema(), seen));
        } else {
          production[--count] =
            generate(wf.schema(), rf.schema(), seen);
        }
      }

      // Add default values for fields missing from Writer
      for (Field rf : rfields) {
        String fname = rf.name();
        Field wf = writer.getField(fname);
        if (wf == null) {
          byte[] bb = getBinary(rf.schema(), rf.defaultValue());
          production[--count] = Symbol.defaultStartAction(bb);
          production[--count] = generate(rf.schema(), rf.schema(), seen);
          production[--count] = Symbol.DEFAULT_END_ACTION;
View Full Code Here

                 FieldMetaData.getStructMetaDataMap(c).values()) {
            Schema s = getSchema(f.valueMetaData);
            if (f.requirementType == TFieldRequirementType.OPTIONAL
                && (s.getType() != Schema.Type.UNION))
              s = nullable(s);
            fields.add(new Field(f.fieldName, s, null, null));
          }
          schema.setFields(fields);
        } else {
          throw new RuntimeException("Not a Thrift-generated class: "+c);
        }
View Full Code Here

    Map<Utf8, Object> currentMap = null;
    List currentArray = null;
    Text currentFam = null;
    int currentPos = 0;
    Schema currentSchema = null;
    Field currentField = null;

    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(new byte[0], null);

    while (iter.hasNext()) {
      Entry<Key,Value> entry = iter.next();

      if (row == null) {
        row = entry.getKey().getRowData();
      }
      byte[] val = entry.getValue().get();

      Field field = fieldMap.get(getFieldName(entry));

      if (currentMap != null) {
        if (currentFam.equals(entry.getKey().getColumnFamily())) {
          currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
              fromBytes(currentSchema, entry.getValue().get()));
          continue;
        } else {
          persistent.put(currentPos, currentMap);
          currentMap = null;
        }
      } else if (currentArray != null) {
        if (currentFam.equals(entry.getKey().getColumnFamily())) {
          currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
          continue;
        } else {
          persistent.put(currentPos, new GenericData.Array<T>(currentField.schema(), currentArray));
          currentArray = null;
        }
      }

      switch (field.schema().getType()) {
      case MAP:  // first entry only. Next are handled above on the next loop
        currentMap = new DirtyMapWrapper<Utf8, Object>(new HashMap<Utf8, Object>());
        currentPos = field.pos();
        currentFam = entry.getKey().getColumnFamily();
        currentSchema = field.schema().getValueType();

        currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
            fromBytes(currentSchema, entry.getValue().get()));
        break;
      case ARRAY:
        currentArray = new DirtyListWrapper<Object>(new ArrayList<Object>());
        currentPos = field.pos();
        currentFam = entry.getKey().getColumnFamily();
        currentSchema = field.schema().getElementType();
        currentField = field;

        currentArray.add(fromBytes(currentSchema, entry.getValue().get()));

        break;
      case UNION:// default value of null acts like union with null
        Schema effectiveSchema = field.schema().getTypes()
        .get(firstNotNullSchemaTypeIndex(field.schema()));
        // map and array were coded without union index so need to be read the same way
        if (effectiveSchema.getType() == Type.ARRAY) {
          currentArray = new DirtyListWrapper<Object>(new ArrayList<Object>());
          currentPos = field.pos();
          currentFam = entry.getKey().getColumnFamily();
          currentSchema = field.schema().getElementType();
          currentField = field;

          currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
          break;
        }
        else if (effectiveSchema.getType() == Type.MAP) {
          currentMap = new DirtyMapWrapper<Utf8, Object>(new HashMap<Utf8, Object>());
          currentPos = field.pos();
          currentFam = entry.getKey().getColumnFamily();
          currentSchema = effectiveSchema.getValueType();

          currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
              fromBytes(currentSchema, entry.getValue().get()));
          break;
        }
        // continue like a regular top-level union
      case RECORD:
        SpecificDatumReader<?> reader = new SpecificDatumReader<Schema>(field.schema());
        persistent.put(field.pos(), reader.read(null, DecoderFactory.get().binaryDecoder(val, decoder)));
        break;
      default:
        persistent.put(field.pos(), fromBytes(field.schema(), entry.getValue().get()));
      }
    }

    if (currentMap != null) {
      persistent.put(currentPos, currentMap);
View Full Code Here

TOP

Related Classes of org.apache.avro.Schema.Field

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.