Examples of org.apache.hadoop.io.serializer.Serialization

org.apache.hadoop.io.serializer.Serialization

Encapsulates a {@link Serializer}/ {@link Deserializer} pair.
@param < T>

   */
  @SuppressWarnings({"unchecked"})
  public OutputCollector getMapperCollector(int mapperIndex,
                                            OutputCollector output,
                                            Reporter reporter) {
    Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
    Serialization valueSerialization =
      mappersValueSerialization.get(mapperIndex);
    return new ChainOutputCollector(mapperIndex, keySerialization,
                                    valueSerialization, output, reporter);
  }

View Full Code Here

          key = makeCopyForPassByValue(keySerialization, key);
          value = makeCopyForPassByValue(valueSerialization, value);
        }


        // gets ser/deser and mapper of next in chain
        Serialization nextKeySerialization =
          mappersKeySerialization.get(nextMapperIndex);
        Serialization nextValueSerialization =
          mappersValueSerialization.get(nextMapperIndex);
        Mapper nextMapper = mappers.get(nextMapperIndex);


        // invokes next mapper in chain
        nextMapper.map(key, value,

View Full Code Here

   */
  @SuppressWarnings({"unchecked"})
  public OutputCollector getMapperCollector(int mapperIndex,
                                            OutputCollector output,
                                            Reporter reporter) {
    Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
    Serialization valueSerialization =
      mappersValueSerialization.get(mapperIndex);
    return new ChainOutputCollector(mapperIndex, keySerialization,
                                    valueSerialization, output, reporter);
  }

View Full Code Here

          key = makeCopyForPassByValue(keySerialization, key);
          value = makeCopyForPassByValue(valueSerialization, value);
        }


        // gets ser/deser and mapper of next in chain
        Serialization nextKeySerialization =
          mappersKeySerialization.get(nextMapperIndex);
        Serialization nextValueSerialization =
          mappersValueSerialization.get(nextMapperIndex);
        Mapper nextMapper = mappers.get(nextMapperIndex);


        // invokes next mapper in chain
        nextMapper.map(key, value,

View Full Code Here

  public static Serializer[] getSerializers(Schema schema, Configuration conf) {
    Serializer[] result = new Serializer[schema.getFields().size()];
    for (int i = 0; i < result.length; i++) {
      Field field = schema.getField(i);
      if (field.getObjectSerialization() != null) {
        Serialization serialization = ReflectionUtils.newInstance(field.getObjectSerialization(), conf);
        if (serialization instanceof FieldConfigurable) {
          ((FieldConfigurable) serialization).setFieldProperties(null, field.getProps());
        }
        result[i] = serialization.getSerializer(field.getObjectClass());
      }
    }
    return result;
  }

View Full Code Here

    for (int i = 0; i < result.length; i++) {
      Field field = readSchema.getField(i);
      if (field.getObjectSerialization() == null) {
        continue;
      }
      Serialization serialization = ReflectionUtils.newInstance(field.getObjectSerialization(), conf);
      if (serialization instanceof FieldConfigurable) {
        Map<String, String> targetSchemaMetadata = null;
        // Look if this field is also in the target Schema, so we extract both metadata
        if(targetSchema.containsField(field.getName())) {
          Field targetSchemaField = targetSchema.getField(field.getName());
          if(targetSchemaField.getObjectSerialization() == null ||
            !targetSchemaField.getObjectSerialization().equals(field.getObjectSerialization())) {
            // Error: field in target schema with same name but different serialization mechanism!
            throw new RuntimeException("Target schema has field [" + field.getName() + "] with different serialization than read schema field with same name.");
          }
          targetSchemaMetadata = targetSchemaField.getProps();
        }
        ((FieldConfigurable) serialization).setFieldProperties(field.getProps(), targetSchemaMetadata);
      }
      result[i] = serialization.getDeserializer(field.getObjectClass());
    }
    return result;
  }

View Full Code Here

   */
  @SuppressWarnings({"unchecked"})
  public OutputCollector getMapperCollector(int mapperIndex,
                                            OutputCollector output,
                                            Reporter reporter) {
    Serialization keySerialization = mappersKeySerialization.get(mapperIndex);
    Serialization valueSerialization =
      mappersValueSerialization.get(mapperIndex);
    return new ChainOutputCollector(mapperIndex, keySerialization,
                                    valueSerialization, output, reporter);
  }

View Full Code Here

          key = makeCopyForPassByValue(keySerialization, key);
          value = makeCopyForPassByValue(valueSerialization, value);
        }


        // gets ser/deser and mapper of next in chain
        Serialization nextKeySerialization =
          mappersKeySerialization.get(nextMapperIndex);
        Serialization nextValueSerialization =
          mappersValueSerialization.get(nextMapperIndex);
        Mapper nextMapper = mappers.get(nextMapperIndex);


        // invokes next mapper in chain
        nextMapper.map(key, value,

View Full Code Here

  public static Serializer[] getSerializers(Schema schema, Configuration conf) {
    Serializer[] result = new Serializer[schema.getFields().size()];
    for (int i = 0; i < result.length; i++) {
      Field field = schema.getField(i);
      if (field.getObjectSerialization() != null) {
        Serialization serialization = ReflectionUtils.newInstance(field.getObjectSerialization(), conf);
        if (serialization instanceof FieldConfigurable) {
          ((FieldConfigurable) serialization).setFieldProperties(field.getProps());
        }
        result[i] = serialization.getSerializer(field.getObjectClass());
      }
    }
    return result;
  }

View Full Code Here

  public static Deserializer[] getDeserializers(Schema schema, Configuration conf) {
    Deserializer[] result = new Deserializer[schema.getFields().size()];
    for (int i = 0; i < result.length; i++) {
      Field field = schema.getField(i);
      if (field.getObjectSerialization() != null) {
        Serialization serialization = ReflectionUtils.newInstance(field.getObjectSerialization(), conf);
        if (serialization instanceof FieldConfigurable) {
          ((FieldConfigurable) serialization).setFieldProperties(field.getProps());
        }
        result[i] = serialization.getDeserializer(field.getObjectClass());
      }
    }
    return result;
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.io.serializer.Serialization

cascading.tuple.hadoop.TupleSerialization

com.datasalt.pangool.tuplemr.SerializationInfo

org.apache.hadoop.mapred.lib.Chain

org.apache.hadoop.mapred.lib.Chain$ChainOutputCollector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.