Examples of DynamicSerDe

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
      Properties schema = new Properties();
      schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
      schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
      schema.setProperty(Constants.SERIALIZATION_DDL,
                         "struct test { i32 hello, list<string> bye, map<string,i32> another}");
      schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());

      schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "false");
      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
       
      hexString(bytes);

      // Try to deserialize
      Object o = serde.deserialize(bytes);
      List<?> olist = (List<?>)o;

      assertTrue(olist.size() == 3);
      assertEquals(new Integer(0), (Integer)olist.get(0));
      List<?> num1 = (List<?>)olist.get(1);
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
      //      schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.thrift.protocol.TJSONProtocol.class.getName());
      schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.thrift.protocol.TBinaryProtocol.class.getName());
      schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
      schema.setProperty(Constants.SERIALIZATION_DDL,
                         "struct inner { i32 field1, string field2 },struct  test {inner foo,  i32 hello, list<string> bye, map<string,i32> another}");
      schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());

     
      //
      // construct object of above type
      //

      // construct the inner struct
      ArrayList<Object> innerStruct = new ArrayList<Object>();
      innerStruct.add(new Integer(22));
      innerStruct.add(new String("hello world"));

      // construct outer struct
      ArrayList<String> bye = new ArrayList<String> ();
      bye.add("firstString");
      bye.add("secondString");
      HashMap<String, Integer> another = new HashMap<String, Integer>(); 
      another.put("firstKey", 1);
      another.put("secondKey", 2);

      ArrayList<Object> struct = new ArrayList<Object>();

      struct.add(innerStruct);
      struct.add(Integer.valueOf(234));
      struct.add(bye);
      struct.add(another);

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);

      // Try to deserialize
      Object o = serde.deserialize(bytes);
      List<?> olist = (List<?>)o;


      assertEquals(4, olist.size());
      assertEquals(innerStruct, olist.get(0));
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
      Properties schema = new Properties();
      schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
      schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
      schema.setProperty(Constants.SERIALIZATION_DDL,
                         "struct test { i32 hello, list<string> bye, map<string,i32> another}");
      schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());

      schema.setProperty(Constants.FIELD_DELIM, "9");
      schema.setProperty(Constants.COLLECTION_DELIM, "1");
      schema.setProperty(Constants.LINE_DELIM, "2");
      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
       
      hexString(bytes);

      String compare = "234" + "\u0009" + "firstString" + "\u0001" + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + "secondKey" + "\u0004" + "2";

      System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize()) + ">");
      System.out.println("compare to    =" + compare + ">");
       
      assertTrue(compare.equals( new String(bytes.get(), 0, bytes.getSize())));
     
      schema.setProperty(Constants.SERIALIZATION_DDL,
                         "struct test { i32 hello, skip list<string> bye, map<string,i32> another}");

      serde.initialize(new Configuration(), schema);

      // Try to deserialize
      Object o = serde.deserialize(bytes);
      System.out.println("o class = " + o.getClass());
      List<?> olist = (List<?>)o;
      System.out.println("o size = " + olist.size());
      System.out.println("o = " + o);
       
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
    //client.execute("select key, count(1) from " + tableName + " where key > 10 group by key");
    String sql = "select key, value from " + tableName + " where key > 10";
    client.execute(sql);

    // Instantiate DynamicSerDe
    DynamicSerDe ds = new DynamicSerDe();
    Properties dsp = new Properties();
    dsp.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
    dsp.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "result");
    String serDDL = new String("struct result { ");
    List<FieldSchema> schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0)
          serDDL = serDDL.concat(",");
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
    dsp.setProperty(Constants.FIELD_DELIM, "9");
    ds.initialize(new Configuration(), dsp);

    String row = client.fetchOne();
    Object o = ds.deserialize(new BytesWritable(row.getBytes()));

    assertEquals(o.getClass().toString(), "class java.util.ArrayList");
    List<?> lst = (List<?>)o;
    assertEquals(lst.get(0), 238);

    // TODO: serde doesn't like underscore  -- struct result { string _c0}
    sql = "select count(1) as c from " + tableName;
    client.execute(sql);
    row = client.fetchOne();

    serDDL = new String("struct result { ");
    schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0)
          serDDL = serDDL.concat(",");
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    // Need a new DynamicSerDe instance - re-initialization is not supported.
    ds = new DynamicSerDe();
    ds.initialize(new Configuration(), dsp);
    o = ds.deserialize(new BytesWritable(row.getBytes()));
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

105
106
107
108
109
110
111
112
113
114
115
        serDDL = serDDL.concat("}");
      }
      else
        serDDL = new String("struct result { string empty }");

      ds = new DynamicSerDe();
      Properties dsp = new Properties();
      dsp.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
      dsp.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "result");
      dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
      dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
        Properties schema = new Properties();
        schema.setProperty(Constants.SERIALIZATION_FORMAT, protocol);
        schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
        schema.setProperty(Constants.SERIALIZATION_DDL,
            "struct test { i32 hello, list<string> bye, map<string,i32> another}");
        schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());
 
        DynamicSerDe serde = new DynamicSerDe();
        serde.initialize(new Configuration(), schema);
       
        // Try getObjectInspector
        ObjectInspector oi = serde.getObjectInspector();
        System.out.println("TypeName = " + oi.getTypeName());

       
        // Try to serialize
        BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
       
        StringBuilder sb = new StringBuilder();
        for (int i=0; i<bytes.getSize(); i++) {
          byte b = bytes.get()[i];
          int v = (b<0 ? 256 + b : b);
          sb.append(String.format("x%02x", v));
        }
        System.out.println("bytes =" + sb);
       
        if (!isBinary) {
          System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize()));
        }
       
        // Try to deserialize
        Object o = serde.deserialize(bytes);
        System.out.println("o class = " + o.getClass());
        List<?> olist = (List<?>)o;
        System.out.println("o size = " + olist.size());
        System.out.println("o[0] class = " + olist.get(0).getClass());
        System.out.println("o[1] class = " + olist.get(1).getClass());
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
      Properties schema = new Properties();
      schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
      schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test");
      schema.setProperty(Constants.SERIALIZATION_DDL,
                         "struct test { i32 hello, list<string> bye, map<string,i32> another}");
      schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString());

      schema.setProperty(Constants.FIELD_DELIM, "9");
      schema.setProperty(Constants.COLLECTION_DELIM, "1");
      schema.setProperty(Constants.LINE_DELIM, "2");
      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator() == 9);
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
       
      StringBuilder sb = new StringBuilder();
      for (int i=0; i<bytes.getSize(); i++) {
        byte b = bytes.get()[i];
        int v = (b<0 ? 256 + b : b);
        sb.append(String.format("x%02x", v));
      }
      System.out.println("bytes =" + sb);

      String compare = "234" + "\u0009" + "firstString" + "\u0001" + "secondString" + "\u0009" + "firstKey" + "\u0004" + "1" + "\u0001" + "secondKey" + "\u0004" + "2";

      System.out.println("bytes in text =" + new String(bytes.get(), 0, bytes.getSize()) + ">");
      System.out.println("compare to    =" + compare + ">");
       
      assertTrue(compare.equals( new String(bytes.get(), 0, bytes.getSize())));
     
      // Try to deserialize
      Object o = serde.deserialize(bytes);
      System.out.println("o class = " + o.getClass());
      List<?> olist = (List<?>)o;
      System.out.println("o size = " + olist.size());
      System.out.println("o[0] class = " + olist.get(0).getClass());
      System.out.println("o[1] class = " + olist.get(1).getClass());
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    // " where key > 10 group by key");
    String sql = "select key, value from " + tableName + " where key > 10";
    client.execute(sql);

    // Instantiate DynamicSerDe
    DynamicSerDe ds = new DynamicSerDe();
    Properties dsp = new Properties();
    dsp.setProperty(Constants.SERIALIZATION_FORMAT,
        org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class
            .getName());
    dsp.setProperty(
        org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,
        "result");
    String serDDL = new String("struct result { ");
    List<FieldSchema> schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
    dsp.setProperty(Constants.FIELD_DELIM, "9");
    ds.initialize(new Configuration(), dsp);

    String row = client.fetchOne();
    Object o = ds.deserialize(new BytesWritable(row.getBytes()));

    assertEquals(o.getClass().toString(), "class java.util.ArrayList");
    List<?> lst = (List<?>) o;
    assertEquals(lst.get(0), 238);

    // TODO: serde doesn't like underscore -- struct result { string _c0}
    sql = "select count(1) as c from " + tableName;
    client.execute(sql);
    row = client.fetchOne();

    serDDL = new String("struct result { ");
    schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    // Need a new DynamicSerDe instance - re-initialization is not supported.
    ds = new DynamicSerDe();
    ds.initialize(new Configuration(), dsp);
    o = ds.deserialize(new BytesWritable(row.getBytes()));
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    // " where key > 10 group by key");
    String sql = "select key, value from " + tableName + " where key > 10";
    client.execute(sql);

    // Instantiate DynamicSerDe
    DynamicSerDe ds = new DynamicSerDe();
    Properties dsp = new Properties();
    dsp.setProperty(serdeConstants.SERIALIZATION_FORMAT,
        org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class
            .getName());
    dsp.setProperty(
        org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,
        "result");
    String serDDL = new String("struct result { ");
    List<FieldSchema> schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(serdeConstants.SERIALIZATION_DDL, serDDL);
    dsp.setProperty(serdeConstants.SERIALIZATION_LIB, ds.getClass().toString());
    dsp.setProperty(serdeConstants.FIELD_DELIM, "9");
    ds.initialize(new Configuration(), dsp);

    String row = client.fetchOne();
    Object o = ds.deserialize(new BytesWritable(row.getBytes()));

    assertEquals(o.getClass().toString(), "class java.util.ArrayList");
    List<?> lst = (List<?>) o;
    assertEquals(lst.get(0), 238);

    // TODO: serde doesn't like underscore -- struct result { string _c0}
    sql = "select count(1) as c from " + tableName;
    client.execute(sql);
    row = client.fetchOne();

    serDDL = new String("struct result { ");
    schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(serdeConstants.SERIALIZATION_DDL, serDDL);
    // Need a new DynamicSerDe instance - re-initialization is not supported.
    ds = new DynamicSerDe();
    ds.initialize(new Configuration(), dsp);
    o = ds.deserialize(new BytesWritable(row.getBytes()));
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe

274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
    // " where key > 10 group by key");
    String sql = "select key, value from " + tableName + " where key > 10";
    client.execute(sql);

    // Instantiate DynamicSerDe
    DynamicSerDe ds = new DynamicSerDe();
    Properties dsp = new Properties();
    dsp.setProperty(Constants.SERIALIZATION_FORMAT,
        org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class
            .getName());
    dsp.setProperty(
        org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,
        "result");
    String serDDL = new String("struct result { ");
    List<FieldSchema> schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
    dsp.setProperty(Constants.FIELD_DELIM, "9");
    ds.initialize(new Configuration(), dsp);

    String row = client.fetchOne();
    Object o = ds.deserialize(new BytesWritable(row.getBytes()));

    assertEquals(o.getClass().toString(), "class java.util.ArrayList");
    List<?> lst = (List<?>) o;
    assertEquals(lst.get(0), 238);

    // TODO: serde doesn't like underscore -- struct result { string _c0}
    sql = "select count(1) as c from " + tableName;
    client.execute(sql);
    row = client.fetchOne();

    serDDL = new String("struct result { ");
    schema = client.getThriftSchema().getFieldSchemas();
    for (int pos = 0; pos < schema.size(); pos++) {
      if (pos != 0) {
        serDDL = serDDL.concat(",");
      }
      serDDL = serDDL.concat(schema.get(pos).getType());
      serDDL = serDDL.concat(" ");
      serDDL = serDDL.concat(schema.get(pos).getName());
    }
    serDDL = serDDL.concat("}");

    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
    // Need a new DynamicSerDe instance - re-initialization is not supported.
    ds = new DynamicSerDe();
    ds.initialize(new Configuration(), dsp);
    o = ds.deserialize(new BytesWritable(row.getBytes()));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.