Examples of org.apache.hadoop.io.serializer.Deserializer

org.apache.hadoop.io.serializer.Deserializer

Provides a facility for deserializing objects of type from an {@link InputStream}.

Deserializers are stateful, but must not buffer the input since other producers may read from the input between calls to {@link #deserialize(Object)}.
@param < T>

   * Return a new instance of the given class with the deserialized data from
   * the input stream.
   */
  public <T> T deser(Class clazz, InputStream in) throws IOException {
    Map<Class, Deserializer> deserializers = cachedDeserializers.get();
    Deserializer deSer = deserializers.get(clazz);
    if(deSer == null) {
      deSer = serialization.getDeserializer(clazz);
      deserializers.put(clazz, deSer);
    }


    deSer.open(in);
    Object obj = deSer.deserialize(null);
    deSer.close();
    return (T) obj;
  }

View Full Code Here

   * Deserialize an object using Hadoop serialization from a byte array. The
   * object cannot be null.
   */
  public <T> T deser(Object obj, byte[] array, int offset, int length) throws IOException {
    Map<Class, Deserializer> deserializers = cachedDeserializers.get();
    Deserializer deSer = deserializers.get(obj.getClass());
    if(deSer == null) {
      deSer = serialization.getDeserializer(obj.getClass());
      deserializers.put(obj.getClass(), deSer);
    }
    DataInputBuffer baIs = cachedInputStream.get();
    baIs.reset(array, offset, length);
    deSer.open(baIs);
    obj = deSer.deserialize(obj);
    deSer.close();
    baIs.close();
    return (T) obj;
  }

View Full Code Here

    inputFormatClass = (Class<? extends InputFormat<?, ?>>) readClass(in);
    mapperClass = (Class<? extends Mapper<?, ?, ?, ?>>) readClass(in);
    inputSplit = (InputSplit) ReflectionUtils
       .newInstance(inputSplitClass, conf);
    SerializationFactory factory = new SerializationFactory(conf);
    Deserializer deserializer = factory.getDeserializer(inputSplitClass);
    deserializer.open((DataInputStream)in);
    inputSplit = (InputSplit)deserializer.deserialize(inputSplit);
  }

View Full Code Here


  Deserializer getNewDeserializer( String className )
    {
    try
      {
      Deserializer deserializer = getSerializationFactory().getDeserializer( getClass( className ) );


      if( deserializer == null )
        throw new CascadingException( "unable to load deserializer for: " + className + " from: " + getSerializationFactory().getClass().getName() );


      return deserializer;

View Full Code Here

      }


    public Object read( int token, DataInputStream inputStream ) throws IOException
      {
      String className = getClassNameFor( token, inputStream );
      Deserializer deserializer = getDeserializerFor( inputStream, className );


      Object foundObject = null;
      Object object;


      try
        {
        object = deserializer.deserialize( foundObject );
        }
      catch( IOException exception )
        {
        LOG.error( "failed deserializing token: " + token + " with classname: " + className, exception );

View Full Code Here

      return tupleSerialization.getComparator( type );
      }


    private Deserializer getDeserializerFor( DataInputStream inputStream, String className ) throws IOException
      {
      Deserializer deserializer = deserializers.get( className );


      if( deserializer == null )
        {
        deserializer = tupleSerialization.getNewDeserializer( className );
        deserializer.open( inputStream );
        deserializers.put( className, deserializer );
        }


      return deserializer;
      }

View Full Code Here


  public void readFields(ITuple tuple, DataInputStream input,Deserializer[] customDeserializers) 
      throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      Deserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()){
      case INT:  tuple.set(index,WritableUtils.readVInt(input)); break;
      case LONG: tuple.set(index,WritableUtils.readVLong(input)); break;
      case DOUBLE: tuple.set(index,input.readDouble()); break;

View Full Code Here

      }
      case BYTES:
        tuple.set(pos,objRecord); //TODO FIXME this should copy bytes really, not reference!
        break;
      case OBJECT:
        Deserializer customDeser = customDeserializers[pos];
        if (objRecord instanceof byte[]){
          inputBuffer.reset((byte[])objRecord,((byte[])objRecord).length);
        } else if (objRecord instanceof ByteBuffer){
          ByteBuffer buffer = (ByteBuffer)objRecord;
          int offset = buffer.arrayOffset()+buffer.position();
          int length = buffer.limit()- buffer.position();
          inputBuffer.reset(buffer.array(),offset,length);
        } else {
          throw new PangoolRuntimeException("Can't convert to OBJECT from instance " + objRecord.getClass());
        }
        if (customDeser != null){
          customDeser.open(inputBuffer);
          tuple.set(pos,customDeser.deserialize(tuple.get(pos))); //TODO FIXME avro deserializer shouldn't reuse objects sometimes (UNION ?)
          customDeser.close(); //TODO is this ok ?
          
        } else {
            //no custom deser , then use Hadoop serializers registered in "io.serializations"
          Class clazz = pangoolField.getObjectClass();
          if(tuple.get(pos) == null || tuple.get(pos).getClass() != clazz) {

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.io.serializer.Deserializer

cascading.tuple.hadoop.TupleSerialization

cascading.tuple.hadoop.TupleSerialization$SerializationElementReader

com.datasalt.pangool.serialization.HadoopSerialization

com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit

com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer

com.datasalt.pangool.tuplemr.serialization.TupleDeserializer

com.datasalt.pangool.utils.AvroRecordToTupleConverter

com.tellapart.cascading.hbase.HBaseSerialization

org.apache.crunch.impl.mr.run.CrunchInputSplit

org.apache.hadoop.mapreduce.lib.input.TaggedInputSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.