Package org.apache.hadoop.io.serializer

Examples of org.apache.hadoop.io.serializer.SerializationFactory

52
53
54
55
56
57
58
    outBuffer = new DataOutputBuffer();
    inBuffer = new DataInputBuffer();

    conf = new Configuration();
    serializationFactory = new SerializationFactory(conf);
  }
View Full Code Here
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
          valLenDecompressor = CodecPool.getDecompressor(codec);
          valLenInFilter = codec.createInputStream(valLenBuffer, valLenDecompressor);
          valLenIn = new DataInputStream(valLenInFilter);
        }
       
        SerializationFactory serializationFactory = new SerializationFactory(conf);
        this.keyDeserializer = getDeserializer(serializationFactory, getKeyClass());
        if (!blockCompressed) {
          this.keyDeserializer.open(valBuffer);
        } else {
          this.keyDeserializer.open(keyIn);
View Full Code Here
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
      this.keyClass = keyClass;
      this.valClass = valClass;
      this.compress = compress;
      this.codec = codec;
      this.metadata = metadata;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keySerializer = serializationFactory.getSerializer(keyClass);
      this.keySerializer.open(buffer);
      this.uncompressedValSerializer = serializationFactory.getSerializer(valClass);
      this.uncompressedValSerializer.open(buffer);
      if (this.codec != null) {
        ReflectionUtils.setConf(this.codec, this.conf);
        this.compressor = CodecPool.getCompressor(this.codec);
        this.deflateFilter = this.codec.createOutputStream(buffer, compressor);
        this.deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
        this.compressedValSerializer = serializationFactory.getSerializer(valClass);
        this.compressedValSerializer.open(deflateOut);
      }
    }
View Full Code Here
496
497
498
499
500
501
502
503
504
505
506
      LOG.info("record buffer = " + softRecordLimit + "/" + kvoffsets.length);
      // k/v serialization
      comparator = job.getOutputKeyComparator();
      keyClass = (Class<K>)job.getMapOutputKeyClass();
      valClass = (Class<V>)job.getMapOutputValueClass();
      serializationFactory = new SerializationFactory(job);
      keySerializer = serializationFactory.getSerializer(keyClass);
      keySerializer.open(bb);
      valSerializer = serializationFactory.getSerializer(valClass);
      valSerializer.open(bb);
      // counters
View Full Code Here
800
801
802
803
804
805
806
807
808
809
810
811
812
813
                           Progressable reporter)
      throws IOException {
      this.in = in;
      this.comparator = comparator;
      this.reporter = reporter;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
      this.keyDeserializer.open(keyIn);
      this.valDeserializer = serializationFactory.getDeserializer(valClass);
      this.valDeserializer.open(this.valueIn);
      readNextKey();
      key = nextKey;
      nextKey = null; // force new instance creation
      hasNext = more;
View Full Code Here
32
33
34
35
36
37
38
  /**
   * @param conf
   */
  public Serialization(Configuration conf) {
    serializationFactory = new SerializationFactory(conf);
  }
View Full Code Here
98
99
100
101
102
103
104
105
   * @param conf
   *          new Configuration object to use
   * @return a new copy of the orig object
   */
  public <T> T copyWithConf(final T orig, final Configuration conf) {
    serializationFactory = new SerializationFactory(conf);
    return copy(orig);
  }
View Full Code Here
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
                           Progressable reporter)
      throws IOException {
      this.in = in;
      this.comparator = comparator;
      this.reporter = reporter;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
      this.keyDeserializer.open(keyIn);
      this.valDeserializer = serializationFactory.getDeserializer(valClass);
      this.valDeserializer.open(this.valueIn);
      readNextKey();
      key = nextKey;
      nextKey = null; // force new instance creation
      hasNext = more;
View Full Code Here
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
      this.ownOutputStream = ownStream;
      this.keyClass = keyClass;
      this.valClass = valClass;
      this.codec = codec;
      this.metadata = metadata;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keySerializer = serializationFactory.getSerializer(keyClass);
      if (this.keySerializer == null) {
        throw new IOException(
            "Could not find a serializer for the Key class: '"
                + keyClass.getCanonicalName() + "'. "
                + "Please ensure that the configuration '" +
                CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
                + "properly configured, if you're using"
                + "custom serialization.");
      }
      this.keySerializer.open(buffer);
      this.uncompressedValSerializer = serializationFactory.getSerializer(valClass);
      if (this.uncompressedValSerializer == null) {
        throw new IOException(
            "Could not find a serializer for the Value class: '"
                + valClass.getCanonicalName() + "'. "
                + "Please ensure that the configuration '" +
                CommonConfigurationKeys.IO_SERIALIZATIONS_KEY + "' is "
                + "properly configured, if you're using"
                + "custom serialization.");
      }
      this.uncompressedValSerializer.open(buffer);
      if (this.codec != null) {
        ReflectionUtils.setConf(this.codec, this.conf);
        this.compressor = CodecPool.getCompressor(this.codec);
        this.deflateFilter = this.codec.createOutputStream(buffer, compressor);
        this.deflateOut =
          new DataOutputStream(new BufferedOutputStream(deflateFilter));
        this.compressedValSerializer = serializationFactory.getSerializer(valClass);
        if (this.compressedValSerializer == null) {
          throw new IOException(
              "Could not find a serializer for the Value class: '"
                  + valClass.getCanonicalName() + "'. "
                  + "Please ensure that the configuration '" +
View Full Code Here
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
          valLenInFilter = codec.createInputStream(valLenBuffer,
                                                   valLenDecompressor);
          valLenIn = new DataInputStream(valLenInFilter);
        }
       
        SerializationFactory serializationFactory =
          new SerializationFactory(conf);
        this.keyDeserializer =
          getDeserializer(serializationFactory, getKeyClass());
        if (this.keyDeserializer == null) {
          throw new IOException(
              "Could not find a deserializer for the Key class: '"
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.serializer.SerializationFactory

  • cascading.tuple.hadoop.TupleSerialization
  • com.datasalt.pangool.serialization.HadoopSerialization
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit
  • com.manning.hip.ch13.TaskSplitReader
  • com.tellapart.cascading.hbase.HBaseSerialization
  • com.twitter.elephantbird.util.SplitUtil
  • com.twitter.maple.tap.TupleMemoryInputFormat
  • org.apache.accumulo.core.file.map.MySequenceFile$Reader
  • org.apache.accumulo.core.file.map.MySequenceFile$Writer
  • org.apache.crunch.impl.mr.run.CrunchInputSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.