Package com.datasalt.pangool.serialization

Examples of com.datasalt.pangool.serialization.HadoopSerialization

107
108
109
110
111
112
113
114
115
116
117
        TupleMRConfig tupleMRConf = builder.buildConf();
        Set<String> instanceFiles = TupleMRConfig.set(tupleMRConf, conf);

        // tupleMRConf has changed -> we need a new Serialization object
        try {
          ser = new HadoopSerialization(conf);
        } catch (Exception e) {
          throw new IOException(e);
        }

        SortComparator sortComparator = new SortComparator();
View Full Code Here
266
267
268
269
270
271
272
273
274
275
    private HadoopSerialization hadoopSer;
    A a1 = new A(), a2 = new A();

    public MyThriftComparator(Configuration conf) {
      try {
        hadoopSer = new HadoopSerialization(conf);
      } catch (IOException e) {
        throw new PangoolRuntimeException(e);
      }
    }
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
  protected void assertSerializable(Tuple tuple, boolean debug) throws IOException {
    assertSerializable(tuple, null, debug);
  }

  protected void assertSerializable(Tuple tuple, Tuple toReuse, boolean debug) throws IOException {
    HadoopSerialization hadoopSer = new HadoopSerialization(getConf());
    SimpleTupleSerializer ser = new SimpleTupleSerializer(hadoopSer);
    SimpleTupleDeserializer deser = new SimpleTupleDeserializer(tuple.getSchema(), hadoopSer, getConf());

    DataOutputBuffer output = new DataOutputBuffer();
    ser.open(output);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
  private final Object instance;
 
 
  public SingleFieldDeserializer(Configuration conf, TupleMRConfig mrConfig,
      Type fieldType,Class<?> objectClazz) throws IOException {
    this.ser = new HadoopSerialization(conf);
    this.fieldType = fieldType;
    this.objectClazz = objectClazz;
    switch(fieldType){
      case STRING: this.instance = new Utf8(); break;
      case OBJECT:
View Full Code Here
75
76
77
78
79
80
81
82
83
84
  public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();
   
    try {
      ser = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
111
112
113
114
115
116
117
    Path file = getDefaultWorkFile(context, "");
    writer
        .create(avroSchema, file.getFileSystem(context.getConfiguration()).create(file));

    return new TupleRecordWriter(avroSchema, pangoolOutputSchema, writer,
        new HadoopSerialization(conf));
  }
View Full Code Here
56
57
58
59
60
61
62
    public TupleInputReader(Configuration conf) throws IOException, InterruptedException {
      specificReader = new SpecificDatumReader<Record>();
      wrapper = new AvroWrapper<Record>();
      this.conf = conf;
      this.ser = new HadoopSerialization(conf);
    }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
  public void setConf(Configuration conf) {
    buf1 = new DataOutputBuffer();
    buf2 = new DataOutputBuffer();
   
    try {
      hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e) {
      throw new RuntimeException(e);
    }
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
  public AvroRecordToTupleConverter(org.apache.avro.Schema avroSchema,Configuration conf){
    this.avroSchema = avroSchema;
    this.pangoolSchema = AvroUtils.toPangoolSchema(avroSchema);
    this.conf = conf;
    try{
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e){
      throw new PangoolRuntimeException(e);
    }
   
    this.customDeserializers = SerializationInfo.getDeserializers(pangoolSchema,conf);
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
  public TupleToAvroRecordConverter(Schema pangoolSchema,Configuration conf){
    this.pangoolSchema = pangoolSchema;
    this.avroSchema = AvroUtils.toAvroSchema(pangoolSchema);
    this.conf = conf;
    try{
      this.hadoopSer = new HadoopSerialization(conf);
    } catch(IOException e){
      throw new PangoolRuntimeException(e);
    }
    this.customSerializers = SerializationInfo.getSerializers(pangoolSchema,conf);
    initBuffers();
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.serialization.HadoopSerialization

  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.io.TupleFile$Reader
  • com.datasalt.pangool.io.TupleFile$Writer
  • com.datasalt.pangool.tuplemr.mapred.BinaryComparator
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleInputFormat$TupleInputReader
  • com.datasalt.pangool.tuplemr.mapred.lib.output.TupleOutputFormat
  • com.datasalt.pangool.tuplemr.mapred.SerializerComparator
  • com.datasalt.pangool.tuplemr.mapred.TestComparators
  • com.datasalt.pangool.tuplemr.mapred.TestComparators$MyThriftComparator
  • com.datasalt.pangool.tuplemr.serialization.SingleFieldDeserializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.