Examples of org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct

Package org.apache.hadoop.hive.serde2.lazybinary

Examples of org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct

org.apache.hadoop.hive.serde2.objectinspector.StructField
LazyBinaryStruct is serialized as follows: start A B A B A B end bytes[] -> |-----|---------|--- ... ---|-----|---------| Section A is one null-byte, corresponding to eight struct fields in Section B. Each bit indicates whether the corresponding field is null (0) or not null (1). Each field is a LazyBinaryObject. Following B, there is another section A and B. This pattern repeats until the all struct fields are serialized.

   * Read and write a randomly generated snappy file.
   * @throws Exception
   */
  @Test
  public void testWithoutIndex() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (InnerStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

      byteValues = new BytesWritable[count];
    }
  }


  private RandomRowInputs writeRandomRows(int count, boolean lowMemoryMode) throws IOException {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (ReallyBigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

    readEveryNthRow(10000, true, NumberOfNulls.NONE);
  }


  private RandomRowInputs writeRandomRowsWithNulls(int count, NumberOfNulls numNulls,
      boolean lowMemoryMode) throws IOException {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (ReallyBigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    ReaderWriterProfiler.setProfilerOptions(conf);

View Full Code Here

    }
  }


  @Test
  public void testMemoryManagement() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (InnerStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  @Test
  /**
   * Test a stride dictionary that contains only the empty string
   */
  public void testEmptyStringStrideDictionary() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

   * Tests writing a stripe containing a string column, which is not dictionary encoded in the
   * first stripe, this is carried over to the third stripe, then dictionary encoding is turned
   * back on.  This will cause the dictionary to be nulled out, then reinitialized.
   */
  public void testStrideDictionariesWithoutStripeCarryover() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

   * skipped, and in the second stride the values in the stride dictionary are read.
   * This can cause problems if seeking across strides is broken for stride dictionary streams.
   * @throws Exception
   */
  public void testSeekAcrossStrideDictionaries() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with a stride dictionary, followed by a stripe without
   * followed by a stripe with.
   */
  public void testEmptyInStringDictionaryStream() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

        return struct;
      }
    };


    // Test control case (cases match)
    StructField field = STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

  public void testCaseInsensitiveFieldsStruct() throws Exception {
    OrcStruct struct = new OrcStruct(Lists.newArrayList(FIELD_0));
    struct.setFieldValue(0, new Text("a"));


    // Test control case (cases match)
    StructField field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.