Examples of org.apache.hadoop.io.serializer.Serializer

org.apache.hadoop.io.serializer.Serializer

Provides a facility for serializing objects of type to an {@link OutputStream}.

Serializers are stateful, but must not buffer the output since other producers may write to the output between calls to {@link #serialize(Object)}.
@param < T>

  public void write(DataOutput out) throws IOException {
    Text.writeString(out, inputSplitClass.getName());
    Text.writeString(out, inputFormatClass.getName());
    Text.writeString(out, mapperClass.getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = 
          factory.getSerializer(inputSplitClass);
    serializer.open((DataOutputStream)out);
    serializer.serialize(inputSplit);
  }

View Full Code Here

      out.writeUTF(e.getValue());
    }
    Text.writeString(out, inputFormatClass.getName());
    Text.writeString(out, inputSplit.getClass().getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = factory.getSerializer(inputSplit.getClass());
    serializer.open((DataOutputStream) out);
    serializer.serialize(inputSplit);
  }

View Full Code Here

        os.writeInt(inputIndex);
        writeObject(targetOps, os);
        os.writeInt(wrappedSplits.length);
        os.writeUTF(wrappedSplits[0].getClass().getName());
        SerializationFactory sf = new SerializationFactory(conf);
        Serializer s = 
            sf.getSerializer(wrappedSplits[0].getClass());
         
        //Checks if Serializer is NULL or not before calling open() method on it.         
        if (s == null) {
              throw new IllegalArgumentException("Could not find Serializer for class "+wrappedSplits[0].getClass()+". InputSplits must implement Writable.");
        }        
        s.open((OutputStream) os);
        for (int i = 0; i < wrappedSplits.length; i++)
        {
            // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
            s.serialize(wrappedSplits[i]);
        }
        
    }

View Full Code Here

  public void write(DataOutput out) throws IOException {
    Text.writeString(out, inputSplitClass.getName());
    Text.writeString(out, inputFormatFile);
    Text.writeString(out, inputProcessorFile);
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = 
          factory.getSerializer(inputSplitClass);
    serializer.open((DataOutputStream)out); 
    serializer.serialize(inputSplit);
  }

View Full Code Here

  /**
   * Serializes the given object using the Hadoop serialization system.
   */
  public void ser(Object datum, OutputStream output) throws IOException {
    Map<Class, Serializer> serializers = cachedSerializers.get();
    Serializer ser = serializers.get(datum.getClass());
    if(ser == null) {
      ser = serialization.getSerializer(datum.getClass());
      if(ser == null) {
        throw new IOException("Serializer for class " + datum.getClass() + " not found");
      }
      serializers.put(datum.getClass(), ser);
    }
    ser.open(output);
    ser.serialize(datum);
    ser.close();
  }

View Full Code Here

      
      List<org.apache.hadoop.mapreduce.InputSplit> splits = input.getSplits(jobContext);
      rawSplits = new ArrayList<RawSplit>(splits.size());
      DataOutputBuffer buffer = new DataOutputBuffer();
      SerializationFactory factory = new SerializationFactory(orbConf);
      Serializer serializer = factory.getSerializer(splits.get(0).getClass());
      serializer.open(buffer);
      for (int i = 0; i < splits.size(); i++) {
        buffer.reset();
        serializer.serialize(splits.get(i));
        RawSplit rawSplit = new RawSplit();
        rawSplit.setClassName(splits.get(i).getClass().getName());
        rawSplit.setDataLength(splits.get(i).getLength());
        rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
        rawSplit.setLocations(splits.get(i).getLocations());

View Full Code Here

  public void write(DataOutput out) throws IOException {
    Text.writeString(out, inputSplitClass.getName());
    Text.writeString(out, inputFormatClass.getName());
    Text.writeString(out, mapperClass.getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = 
          factory.getSerializer(inputSplitClass);
    serializer.open((DataOutputStream)out);
    serializer.serialize(inputSplit);
  }

View Full Code Here


  Serializer getNewSerializer( Class type )
    {
    try
      {
      Serializer serializer = getSerializationFactory().getSerializer( type );


      if( serializer == null )
        throw new CascadingException( "unable to load serializer for: " + type.getName() + " from: " + getSerializationFactory().getClass().getName() );


      return serializer;

View Full Code Here

      else
        {
        WritableUtils.writeVInt( outputStream, token );
        }


      Serializer serializer = serializers.get( type );


      if( serializer == null )
        {
        serializer = tupleSerialization.getNewSerializer( type );
        serializer.open( outputStream );
        serializers.put( type, serializer );
        }


      try
        {
        serializer.serialize( object );
        }
      catch( IOException exception )
        {
        LOG.error( "failed serializing token: " + token + " with classname: " + className, exception );

View Full Code Here

    for(int i = 0; i < c.getElements().size(); i++) {
      Field field = schema.getField(i);
      SortElement e = c.getElements().get(i);
      Object o1 = w1.get(index1[i]);
      Object o2 = w2.get(index2[i]);
      Serializer serializer = (serializers == null) ? null : serializers[i];
      int comparison = compareObjects(o1, o2, e.getCustomComparator(), field.getType(),serializer);
      if(comparison != 0) {
        return(e.getOrder() == Order.ASC ? comparison : -comparison);
      }
    }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.io.serializer.Serializer

cascading.tuple.hadoop.TupleSerialization

cascading.tuple.hadoop.TupleSerialization$SerializationElementWriter

com.datasalt.pangool.serialization.HadoopSerialization

com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit

com.datasalt.pangool.tuplemr.mapred.SortComparator

com.datasalt.pangool.utils.TupleToAvroRecordConverter

com.tellapart.cascading.hbase.HBaseSerialization

com.twitter.elephantbird.util.SplitUtil

org.apache.crunch.impl.mr.run.CrunchInputSplit

org.apache.hadoop.hive.ql.io.TestFlatFileInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.