Package org.apache.hadoop.hive.serde2

Examples of org.apache.hadoop.hive.serde2.CustomTextSerDe

2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
  /**
   * Tests a writing a stripe with a stride dictionary, followed by a stripe without
   * followed by a stripe with.
   */
  public void testEmptyInIntDictionaryStream() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
   *
   * @throws Exception
   */
  @Test
  public void testSkipWithEmptyArrayInEnd() throws Exception {
    ObjectInspector inspector;
    List<String> emptyList = Collections.emptyList();
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringListWithId.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
  /**
   * Tests a writing a stripe with an integer column, which enters low memory mode before the first
   * index stride is complete.
   */
  public void testIntEnterLowMemoryModeInFirstStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
  /**
   * Tests a writing a stripe with a string column, which enters low memory mode before the first
   * index stride is complete.
   */
  public void testStringEnterLowMemoryModeInFirstStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    public void close(boolean b) throws IOException {
      // if we haven't written any rows, we need to create a file with a
      // generic schema.
      if (writer == null) {
        // a row with no columns
        ObjectInspector inspector = ObjectInspectorFactory.
            getStandardStructObjectInspector(new ArrayList<String>(),
                new ArrayList<ObjectInspector>());
        writer = OrcFile.createWriter(fs, path, this.conf, inspector,
            stripeSize, compress, compressionSize, rowIndexStride);
      }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    checkOutput(expectedFileUrl.getPath(), outputFilename);
  }

  @Test
  public void testDump() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    writer.close();
    checkOutput("orc-file-dump.out");
  }

  private void testDictionary(Configuration conf, String expectedOutputFilename) throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    // Turn off using the approximate entropy heuristic to turn off dictionary encoding
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
  //Test that if the number of distinct characters in distinct strings is less than the configured
  // threshold dictionary encoding is turned off.  If dictionary encoding is turned off the length
  // of the dictionary stream for the column will be 0 in the ORC file dump.
  @Test
  public void testEntropyThreshold() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    OrcConf.setFloatVar(conf, OrcConf.ConfVars.HIVE_ORC_ENTROPY_KEY_STRING_SIZE_THRESHOLD, 1);
View Full Code Here
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
        return struct;
      }
    };

    // Test control case (cases match)
    StructField field = STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
  }
View Full Code Here
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
  public void testCaseInsensitiveFieldsStruct() throws Exception {
    OrcStruct struct = new OrcStruct(Lists.newArrayList(FIELD_0));
    struct.setFieldValue(0, new Text("a"));

    // Test control case (cases match)
    StructField field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.CustomTextSerDe

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.