Examples of org.apache.hadoop.io.serializer.SerializationFactory

org.apache.hadoop.io.serializer.SerializationFactory

A factory for {@link Serialization}s.


  /**
   * @param conf
   */
  public Serialization(Configuration conf) {
    serializationFactory = new SerializationFactory(conf);
  }

View Full Code Here

   * @param conf
   *          new Configuration object to use
   * @return a new copy of the orig object
   */
  public <T> T copyWithConf(final T orig, final Configuration conf) {
    serializationFactory = new SerializationFactory(conf);
    return copy(orig);
  }

View Full Code Here

    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).readFields(in);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Deserializer deserializer = factory.getDeserializer(inputSplitClass);
      deserializer.open((DataInputStream) in);
      inputSplit = (InputSplit) deserializer.deserialize(inputSplit);
      deserializer.close();
    }
  }

View Full Code Here

    bundle.write(out);
    Text.writeString(out, inputSplit.getClass().getName());
    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).write(out);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Serializer serializer = factory.getSerializer(inputSplit.getClass());
      serializer.open((OutputStream) out);
      serializer.serialize(inputSplit);
      serializer.close();
    }
  }

View Full Code Here

        String[] distinctSplitClassName = new String[distinctSplitClassCount];
        for (int i = 0; i < distinctSplitClassCount; i++) {
            distinctSplitClassName[i] = is.readUTF();
        }
        try {
            SerializationFactory sf = new SerializationFactory(conf);
            // The correct call sequence for Deserializer is, we shall open, then deserialize, but we shall not close
            wrappedSplits = new InputSplit[splitLen];
            for (int i = 0; i < splitLen; i++)
            {
                //read the className index
                int index = is.readInt();
                //get the split class name
                String splitClassName = distinctSplitClassName[index];
                Class splitClass = conf.getClassByName(splitClassName);
                Deserializer d = sf.getDeserializer(splitClass);
                d.open((InputStream) is);
                wrappedSplits[i] = (InputSplit)ReflectionUtils.newInstance(splitClass, conf);
                d.deserialize(wrappedSplits[i]);
            }
        } catch (ClassNotFoundException e) {

View Full Code Here

        os.writeInt(distinctSplitClassList.size());
        //write each classname once
        for (int i = 0 ; i < distinctSplitClassList.size(); i++) {
            os.writeUTF(distinctSplitClassList.get(i));
        }
        SerializationFactory sf = new SerializationFactory(conf);


        for (int i = 0; i < wrappedSplits.length; i++)
        {
            //find out the index of the split class name
            int index = distinctSplitClassList.indexOf(wrappedSplits[i].getClass().getName());
            os.writeInt(index);
            Serializer s = sf.getSerializer(wrappedSplits[i].getClass());
            //Checks if Serializer is NULL or not before calling open() method on it.
            if (s == null) {
                throw new IllegalArgumentException("Could not find Serializer for class "+wrappedSplits[i].getClass()+". InputSplits must implement Writable.");
            }
            s.open((OutputStream) os);

View Full Code Here

          valLenInFilter = codec.createInputStream(valLenBuffer, 
                                                   valLenDecompressor);
          valLenIn = new DataInputStream(valLenInFilter);
        }
        
        SerializationFactory serializationFactory =
          new SerializationFactory(conf);
        this.keyDeserializer =
          getDeserializer(serializationFactory, getKeyClass());
        if (!blockCompressed) {
          this.keyDeserializer.open(valBuffer);
        } else {

View Full Code Here

        this.out = new FSDataOutputStream(checksumOut,null);
      }
      
      this.keyClass = keyClass;
      this.valueClass = valueClass;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keySerializer = serializationFactory.getSerializer(keyClass);
      this.keySerializer.open(buffer);
      this.valueSerializer = serializationFactory.getSerializer(valueClass);
      this.valueSerializer.open(buffer);
    }

View Full Code Here

     IOException wrap = new IOException("Split class " + className + 
                                         " not found");
     wrap.initCause(ce);
     throw wrap;
   }
   SerializationFactory factory = new SerializationFactory(conf);
   Deserializer<T> deserializer = 
     (Deserializer<T>) factory.getDeserializer(cls);
   deserializer.open(inFile);
   T split = deserializer.deserialize(null);
   long pos = inFile.getPos();
   getCounters().findCounter(
       TaskCounter.SPLIT_RAW_BYTES).increment(pos - offset);

View Full Code Here


      // k/v serialization
      comparator = job.getOutputKeyComparator();
      keyClass = (Class<K>)job.getMapOutputKeyClass();
      valClass = (Class<V>)job.getMapOutputValueClass();
      serializationFactory = new SerializationFactory(job);
      keySerializer = serializationFactory.getSerializer(keyClass);
      keySerializer.open(bb);
      valSerializer = serializationFactory.getSerializer(valClass);
      valSerializer.open(bb);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.serializer.SerializationFactory

cascading.tuple.hadoop.TupleSerialization

com.datasalt.pangool.serialization.HadoopSerialization

com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit

com.manning.hip.ch13.TaskSplitReader

com.tellapart.cascading.hbase.HBaseSerialization

com.twitter.elephantbird.util.SplitUtil

com.twitter.maple.tap.TupleMemoryInputFormat

org.apache.accumulo.core.file.map.MySequenceFile$Reader

org.apache.accumulo.core.file.map.MySequenceFile$Writer

org.apache.crunch.impl.mr.run.CrunchInputSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.