Package com.datasalt.pangool.serialization

Examples of com.datasalt.pangool.serialization.HadoopSerialization

88
89
90
91
92
93
94
95
96
97
  public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();
   
    try {
      hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
  public AvroRecordToTupleConverter(org.apache.avro.Schema avroSchema,Configuration conf){
    this.avroSchema = avroSchema;
    this.pangoolSchema = AvroUtils.toPangoolSchema(avroSchema);
    this.conf = conf;
    try{
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e){
      throw new PangoolRuntimeException(e);
    }
   
    this.customDeserializers = SerializationInfo.getDeserializers(pangoolSchema);
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
  public TupleToAvroRecordConverter(Schema pangoolSchema,Configuration conf){
    this.pangoolSchema = pangoolSchema;
    this.avroSchema = AvroUtils.toAvroSchema(pangoolSchema);
    this.conf = conf;
    try{
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e){
      throw new PangoolRuntimeException(e);
    }
    this.customSerializers = SerializationInfo.getSerializers(pangoolSchema);
    initBuffers();
View Full Code Here
62
63
64
65
66
67
68
69
70
71
  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
    try {
      this.ser = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
  public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();
   
    try {
      ser = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
115
116
117
118
119
120
121
    Path file = getDefaultWorkFile(context, "");
    writer
        .create(avroSchema, file.getFileSystem(context.getConfiguration()).create(file));

    return new TupleRecordWriter(avroSchema, pangoolOutputSchema, writer,
        new HadoopSerialization(conf));
  }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
      innerWriter = SequenceFile.createWriter(fs, conf, name, UNUSED, UNUSED, compressionType, codec, progress,metadata);
      init(conf, schema);
    }

    private void init(Configuration conf, Schema schema) throws IOException {
      HadoopSerialization hadoopSer = new HadoopSerialization(conf);
      this.ser = new SimpleTupleSerializer(schema, hadoopSer, conf);
      this.outputBuffer = new DataOutputBuffer();
      ser.open(outputBuffer);
    }
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
    public Reader(FileSystem fs, Schema targetSchema, Configuration conf, Path file) throws IOException {
      this.file = file;
      innerReader = new SequenceFile.Reader(fs, file, conf);
      loadSchema();

      HadoopSerialization ser = new HadoopSerialization(conf);
      if(targetSchema != null) {
        this.deser = new SimpleTupleDeserializer(schema, targetSchema, ser, conf);
        this.tuple = new Tuple(targetSchema);
      } else {
        this.deser = new SimpleTupleDeserializer(schema, ser, conf);       
View Full Code Here
77
78
79
80
81
82
83
84
85
86
  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
    try {
      this.ser = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
  public AvroRecordToTupleConverter(org.apache.avro.Schema avroSchema, Configuration conf) {
    this.avroSchema = avroSchema;
    this.pangoolSchema = AvroUtils.toPangoolSchema(avroSchema);
    this.conf = conf;
    try {
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new PangoolRuntimeException(e);
    }

    this.customDeserializers = SerializationInfo.getDeserializers(pangoolSchema, conf);
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.serialization.HadoopSerialization

  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.io.TupleFile$Reader
  • com.datasalt.pangool.io.TupleFile$Writer
  • com.datasalt.pangool.tuplemr.mapred.BinaryComparator
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.SerializerComparator
  • com.datasalt.pangool.tuplemr.mapred.TestComparators
  • com.datasalt.pangool.tuplemr.mapred.TestComparators$MyThriftComparator
  • com.datasalt.pangool.tuplemr.serialization.SingleFieldDeserializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.