Package org.apache.hadoop.io.serializer

Examples of org.apache.hadoop.io.serializer.Deserializer

222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
        String splitClassName = is.readUTF();
        try {
            Class splitClass = conf.getClassByName(splitClassName);
            SerializationFactory sf = new SerializationFactory(conf);
            // The correct call sequence for Deserializer is, we shall open, then deserialize, but we shall not close
            Deserializer d = sf.getDeserializer(splitClass);
            d.open((InputStream) is);
            wrappedSplits = new InputSplit[splitLen];
            for (int i = 0; i < splitLen; i++)
            {
                wrappedSplits[i] = (InputSplit)ReflectionUtils.newInstance(splitClass, conf);
                d.deserialize(wrappedSplits[i]);
            }
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }
       
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
      }
      case BYTES:
        tuple.set(pos, objRecord); // TODO FIXME this should copy bytes really, not reference!
        break;
      case OBJECT:
        Deserializer customDeser = customDeserializers[pos];
        if(objRecord instanceof byte[]) {
          inputBuffer.reset((byte[]) objRecord, ((byte[]) objRecord).length);
        } else if(objRecord instanceof ByteBuffer) {
          ByteBuffer buffer = (ByteBuffer) objRecord;
          int offset = buffer.arrayOffset() + buffer.position();
          int length = buffer.limit() - buffer.position();
          inputBuffer.reset(buffer.array(), offset, length);
        } else {
          throw new PangoolRuntimeException("Can't convert to OBJECT from instance " + objRecord.getClass());
        }
        if(customDeser != null) {
          customDeser.open(inputBuffer);
          tuple.set(pos, customDeser.deserialize(tuple.get(pos))); // TODO FIXME avro deserializer shouldn't reuse
                                                                   // objects sometimes (UNION ?)
          customDeser.close(); // TODO is this ok ?
        } else {
          // no custom deser , then use Hadoop serializers registered in "io.serializations"
          Class clazz = pangoolField.getObjectClass();
          if(tuple.get(pos) == null || tuple.get(pos).getClass() != clazz) {
            tuple.set(pos, ReflectionUtils.newInstance(clazz, conf));
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  }

  public void readFields(ITuple tuple, Deserializer[] customDeserializers) throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      Deserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()) {
      case INT:
        tuple.set(index, WritableUtils.readVInt(input));
        break;
View Full Code Here
120
121
122
123
124
125
126
127
128
    inputFormatClass = (Class<? extends InputFormat<?, ?>>) readClass(in);
    mapperClass = (Class<? extends Mapper<?, ?, ?, ?>>) readClass(in);
    inputSplit = (InputSplit) ReflectionUtils
       .newInstance(inputSplitClass, conf);
    SerializationFactory factory = new SerializationFactory(conf);
    Deserializer deserializer = factory.getDeserializer(inputSplitClass);
    deserializer.open((DataInputStream)in);
    inputSplit = (InputSplit)deserializer.deserialize(inputSplit);
  }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
      }
    }

    // Field by field deseralization
    for(int index = 0; index < schema.getFields().size(); index++) {
      Deserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);

      // Nulls control
      if (field.isNullable() && nullsAbsolute.flags[index]) {
        // Null field. Nothing to deserialize.
View Full Code Here
88
89
90
91
92
93
94
95
96
    }
    inputFormatClass = (Class<? extends InputFormat<?, ?>>) readClass(in);
    Class<? extends InputSplit> inputSplitClass = (Class<? extends InputSplit>) readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
    SerializationFactory factory = new SerializationFactory(conf);
    Deserializer deserializer = factory.getDeserializer(inputSplitClass);
    deserializer.open((DataInputStream) in);
    inputSplit = (InputSplit) deserializer.deserialize(inputSplit);
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
  }

  public void readFields(ITuple tuple, Deserializer[] customDeserializers) throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      Deserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()) {
      case INT:
        tuple.set(index, WritableUtils.readVInt(input));
        break;
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
        String splitClassName = is.readUTF();
        try {
            Class splitClass = conf.getClassByName(splitClassName);
            SerializationFactory sf = new SerializationFactory(conf);
            // The correct call sequence for Deserializer is, we shall open, then deserialize, but we shall not close
            Deserializer d = sf.getDeserializer(splitClass);
            d.open((InputStream) is);
            wrappedSplits = new InputSplit[splitLen];
            for (int i = 0; i < splitLen; i++)
            {
                wrappedSplits[i] = (InputSplit)ReflectionUtils.newInstance(splitClass, conf);
                d.deserialize(wrappedSplits[i]);
            }
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }
       
View Full Code Here
103
104
105
106
107
108
109
110
111
    inputFormatFile = Text.readString(in);
    inputProcessorFile = Text.readString(in);
    inputSplit = (InputSplit) ReflectionUtils
       .newInstance(inputSplitClass, conf);
    SerializationFactory factory = new SerializationFactory(conf);
    Deserializer deserializer = factory.getDeserializer(inputSplitClass);
    deserializer.open((DataInputStream)in);
    inputSplit = (InputSplit)deserializer.deserialize(inputSplit);
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
   * Deseerializes into the given object using the Hadoop serialization system.
   * Object cannot be null.
   */
  public <T> T deser(Object obj, InputStream in) throws IOException {
    Map<Class, Deserializer> deserializers = cachedDeserializers.get();
    Deserializer deSer = deserializers.get(obj.getClass());
    if(deSer == null) {
      deSer = serialization.getDeserializer(obj.getClass());
      deserializers.put(obj.getClass(), deSer);
    }
    deSer.open(in);
    obj = deSer.deserialize(obj);
    deSer.close();
    return (T) obj;
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.serializer.Deserializer

  • cascading.tuple.hadoop.TupleSerialization
  • cascading.tuple.hadoop.TupleSerialization$SerializationElementReader
  • com.datasalt.pangool.serialization.HadoopSerialization
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit
  • com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer
  • com.datasalt.pangool.tuplemr.serialization.TupleDeserializer
  • com.datasalt.pangool.utils.AvroRecordToTupleConverter
  • com.tellapart.cascading.hbase.HBaseSerialization
  • org.apache.crunch.impl.mr.run.CrunchInputSplit
  • org.apache.hadoop.mapreduce.lib.input.TaggedInputSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.