Package com.datasalt.pangool.serialization

Examples of com.datasalt.pangool.serialization.HadoopSerialization

48
49
50
51
52
53
54
55
56
57
58
 
  public TupleToAvroRecordConverter(Schema pangoolSchema,Configuration conf){
    this.pangoolSchema = pangoolSchema;
    this.avroSchema = AvroUtils.toAvroSchema(pangoolSchema);
    try{
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e){
      throw new PangoolRuntimeException(e);
    }
    this.customSerializers = SerializationInfo.getSerializers(pangoolSchema,conf);
    initBuffers();
View Full Code Here
89
90
91
92
93
94
95
96
97
98
  public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();
   
    try {
      hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
180
181
182
183
184
185
186
187
188
189
      innerWriter = SequenceFile.createWriter(fs, conf, name, UNUSED, UNUSED, compressionType, codec, progress,metadata);
      init(conf, schema);
    }

    private void init(Configuration conf, Schema schema) throws IOException {
      HadoopSerialization hadoopSer = new HadoopSerialization(conf);
      this.ser = new SimpleTupleSerializer(schema, hadoopSer, conf);
      this.outputBuffer = new DataOutputBuffer();
      ser.open(outputBuffer);
    }
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
      this.file = file;
      this.conf = conf;
      innerReader = new SequenceFile.Reader(fs, file, conf);
      loadSchema();

      HadoopSerialization ser = new HadoopSerialization(conf);
      this.deser = new SimpleTupleDeserializer(schema, ser, conf);
      this.tuple = new Tuple(schema);

      deser.open(inBuffer);
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
    try {
      this.ser = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
      innerWriter = SequenceFile.createWriter(fs, conf, name, UNUSED, UNUSED, compressionType, codec, progress,metadata);
      init(conf, schema);
    }

    private void init(Configuration conf, Schema schema) throws IOException {
      HadoopSerialization hadoopSer = new HadoopSerialization(conf);
      this.ser = new SimpleTupleSerializer(schema, hadoopSer, conf);
      this.outputBuffer = new DataOutputBuffer();
      ser.open(outputBuffer);
    }
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
        throws IOException {
      this.file = file;
      innerReader = new SequenceFile.Reader(fs, file, conf);
      loadSchema();

      HadoopSerialization ser = new HadoopSerialization(conf);
      this.deser = new SimpleTupleDeserializer(schema, ser, conf);
      this.tuple = new Tuple(schema);

      deser.open(inBuffer);
    }
View Full Code Here
25
26
27
28
29
30
31
32
33
34
35
  public void test() throws IOException {
    Schema schema = new Schema("schema", Fields.parse("a:string, b:int?, c:double"));
    Schema targetSchema = new Schema("target", Fields.parse("a:string, b:int?, c:double, d:long?, e:boolean?"));
   
    Configuration conf = new Configuration();
    HadoopSerialization hadoopSerDe = new HadoopSerialization(conf);

    ITuple tuple = new Tuple(schema);
    tuple.set("a", "foo");
    tuple.set("b", 10);
    tuple.set("c", 5d);
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
   
    Schema schema = new Schema("schema", Fields.parse("a:string, b:int?"));
    Schema targetSchema = Mutator.superSetOf(schema, cField, dField);
   
    Configuration conf = new Configuration();
    HadoopSerialization hadoopSerDe = new HadoopSerialization(conf);

    ITuple tuple = new Tuple(schema);
    tuple.set("a", "foo");
    tuple.set("b", 10);
   
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
  public void testRandomTupleSerialization(boolean withNulls) throws IOException, TupleMRException {
    Configuration conf = getConf();
    //ThriftSerialization.enableThriftSerialization(conf);

    HadoopSerialization hadoopSer = new HadoopSerialization(conf);
    //defined in BaseTest
    TupleMRConfig pangoolConf = buildPangoolConfig(withNulls);
    List<Schema> intermediateSchemas = pangoolConf.getIntermediateSchemas();

    TupleSerialization serialization = new TupleSerialization(hadoopSer, pangoolConf);
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.serialization.HadoopSerialization

  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.io.TupleFile$Reader
  • com.datasalt.pangool.io.TupleFile$Writer
  • com.datasalt.pangool.tuplemr.mapred.BinaryComparator
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.SerializerComparator
  • com.datasalt.pangool.tuplemr.mapred.TestComparators
  • com.datasalt.pangool.tuplemr.mapred.TestComparators$MyThriftComparator
  • com.datasalt.pangool.tuplemr.serialization.SingleFieldDeserializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.