Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.LazyMap

35
36
37
38
39
40
41
42
    super(PrimitiveObjectInspectorUtils.timestampTypeEntry);
  }

  @Override
  public Timestamp getPrimitiveJavaObject(Object o) {
    TimestampWritable writable = getPrimitiveWritableObject(o);
    return writable == null ? null : writable.getTimestamp();
  }
View Full Code Here
521
522
523
524
525
526
527
528
529
530
531
      int nextCalls = 0;

      @Override
      public Object next(Object previous) throws IOException {
        if (nextCalls == 0) {
          return new TimestampWritable(new Timestamp(1));
        }

        throw new IOException("next should only be called once");
      }
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    return nanos.loadIndeces(rowIndexEntries, updatedStartIndex);
  }

  @Override
  public Object next(Object previous) throws IOException {
    TimestampWritable result = null;
    if (valuePresent) {
      if (previous == null) {
        result = new TimestampWritable();
      } else {
        result = (TimestampWritable) previous;
      }
      long millis = (data.next() + WriterImpl.BASE_TIMESTAMP) *
          WriterImpl.MILLIS_PER_SECOND;
      int newNanos = parseNanos(nanos.next());
      // fix the rounding when we divided by 1000.
      if (millis >= 0) {
        millis += newNanos / 1000000;
      } else {
        millis -= newNanos / 1000000;
      }
      Timestamp timestamp = result.getTimestamp();
      timestamp.setTime(millis);
      timestamp.setNanos(newNanos);
      result.set(timestamp);
    }
    return result;
  }
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
        Object o =  struct.getField(0);
        assertEquals(LazyString.class, o.getClass());
        o =  struct.getField(1);
        assertEquals(LazyMap.class, o.getClass());

        LazyMap arr = (LazyMap)o;
        Map<Object,Object> values = arr.getMap();
        for(Entry<Object,Object> entry : values.entrySet()) {
            assertEquals(LazyString.class, entry.getKey().getClass());
            assertEquals(LazyString.class, entry.getValue().getClass());

            String keyStr =((LazyString) entry.getKey()).getWritableObject().toString();
View Full Code Here
234
235
236
237
238
239
240
241
242
243
244
245
246
247
    assertEquals(row.getRowId(), ((LazyString) field0).getWritableObject().toString());

    Object field1 = lazyRow.getField(1);
    assertNotNull(field1);
    assertTrue(field1 instanceof LazyMap);
    LazyMap map = (LazyMap) field1;

    Map<Object,Object> untypedMap = map.getMap();
    assertEquals(3, map.getMapSize());
    Set<String> expectedKeys = new HashSet<String>();
    expectedKeys.add("k1");
    expectedKeys.add("k2");
    expectedKeys.add("k3");
    for (Entry<Object,Object> entry : untypedMap.entrySet()) {
View Full Code Here
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
    LazyStruct rowStruct = (LazyStruct) objects.get(0);
    List<Object> rowObjects = rowStruct.getFieldsAsList();
    Assert.assertEquals(2, rowObjects.size());

    LazyMap rowMap = (LazyMap) rowObjects.get(0);
    Map<?,?> actualMap = rowMap.getMap();
    System.out.println("Actual map 1: " + actualMap);
    Map<String,String> actualStringMap = new HashMap<String,String>();
    for (Entry<?,?> entry : actualMap.entrySet()) {
      actualStringMap.put(entry.getKey().toString(), entry.getValue().toString());
    }

    Assert.assertEquals(map1, actualStringMap);

    rowMap = (LazyMap) rowObjects.get(1);
    actualMap = rowMap.getMap();
    System.out.println("Actual map 2: " + actualMap);
    actualStringMap = new HashMap<String,String>();
    for (Entry<?,?> entry : actualMap.entrySet()) {
      actualStringMap.put(entry.getKey().toString(), entry.getValue().toString());
    }
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
        return deserializeStruct(rowField, f.getFieldName());

      } else if (rowField instanceof LazyMap) {
        // We have found a map. Systematically deserialize the values of the map and return back the
        // map
        LazyMap lazyMap = (LazyMap) rowField;

        for (Entry<Object, Object> entry : lazyMap.getMap().entrySet()) {
          Object _key = entry.getKey();
          Object _value = entry.getValue();

          if (_value instanceof LazyStruct) {
            lazyMap.getMap().put(_key, deserializeStruct(_value, f.getFieldName()));
          }
        }

        if (LOG.isDebugEnabled()) {
          LOG.debug("Returning a lazy map for field [" + f.getFieldName() + "]");
View Full Code Here
422
423
424
425
426
427
428
429
430
431
432
433
434
    }

    // avro guarantees that the key will be of type string. So we just need to worry about
    // deserializing the value here

    LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector);

    Map map = lazyMap.getMap();

    Map<Object, Object> origMap = (Map) obj;

    ObjectInspector keyObjectInspector =
        ((MapObjectInspector) objectInspector).getMapKeyObjectInspector();
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        assertMapDecode("\\N\u0003ignored\u0002null\u0003\\N", expectedMap);
    }

    public static void assertMapDecode(String encodedMap, Map<? extends Object, ? extends Object> expectedMap)
    {
        LazyMap lazyMap = (LazyMap) createLazyObject(getLazySimpleMapObjectInspector(
                LAZY_STRING_OBJECT_INSPECTOR,
                getLazyStringObjectInspector(false, (byte) 0),
                (byte) 2,
                (byte) 3,
                new Text("\\N"),
                false,
                (byte) 0
        ));

        lazyMap.init(newByteArrayRef(encodedMap), 0, encodedMap.length());

        Map<Object, Object> map = lazyMap.getMap();
        assertEquals(map, expectedMap);
    }
View Full Code Here
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
                   StreamFactory writer,
                   boolean nullable, Configuration conf,
                   boolean useVInts, boolean lowMemoryMode,
                   MemoryEstimate memoryEstimate) throws IOException {
      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
      ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
      childrenWriters = new TreeWriter[1];
      childrenWriters[0] =
        createTreeWriter(listObjectInspector.getListElementObjectInspector(),
          writer, true, conf, useVInts, lowMemoryMode, memoryEstimate);
      lengths =
        new RunLengthIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
      recordPosition(rowIndexPosition);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.LazyMap

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.