Examples of com.facebook.hive.orc.OrcTestUtils.StringStruct

Package com.facebook.hive.orc.OrcTestUtils

Examples of com.facebook.hive.orc.OrcTestUtils.StringStruct

com.facebook.hive.orc.OrcTestUtils.StringStruct

    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_DICTIONARY_SORT_KEYS, true);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    // Write a stripe which is not dictionary encoded
    for (int i = 0; i < 2000; i++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }
    memory.forceFlushStripe();
    // Write another stripe (doesn't matter what)
    for (int i = 0; i < 2000; i++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }
    memory.forceFlushStripe();
    // Write a stripe which will be dictionary encoded
    // Note: it is important that this string is lexicographically after the string in the next
    // index stride.  This way, if sorting by index strides is not working, this value will appear
    // after the next one, though it should appear before, yielding incorrect results.
    writer.addRow(new StringStruct("b"));
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    writer.addRow(new StringStruct("a"));
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    memory.forceFlushStripe();
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    OrcConf.setIntVar(conf, OrcConf.ConfVars.HIVE_ORC_ENTROPY_STRING_THRESHOLD, 1);
    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_BUILD_STRIDE_DICTIONARY, true);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    // Write this value once, so it's added to a stride dictionary
    writer.addRow(new StringStruct("a"));
    // Fill out the rest of the stride
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    // Write this value once, so it's added to a stride dictionary
    writer.addRow(new StringStruct("b"));
    // Fill out the rest of the stride
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    writer.close();


    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    MemoryManagerWithForce memory = new MemoryManagerWithForce(conf);
    ReaderWriterProfiler.setProfilerOptions(conf);
    OrcConf.setIntVar(conf, OrcConf.ConfVars.HIVE_ORC_ENTROPY_STRING_THRESHOLD, 1);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    writer.addRow(new StringStruct("a"));
    writer.addRow(new StringStruct("b"));
    writer.addRow(new StringStruct("c"));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new StringStruct("123"));
    }
    memory.forceFlushStripe();
    for (int i = 0; i < 1000; i++) {
      writer.addRow(new StringStruct("123"));
    }
    memory.forceFlushStripe();
    writer.addRow(new StringStruct("a"));
    writer.addRow(new StringStruct("b"));
    writer.addRow(new StringStruct("c"));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new StringStruct("123"));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;

View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 10000, memory);


    // Write 500 rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Force the writer to enter low memory mode, note since the stride length was set to 10000
    // we're still in the first stride
    memory.forceEnterLowMemoryMode();


    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i + 500)));
    }


    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);


    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Write 250 more rows (a portion of the second stride)
    for (int i = 0; i < 250; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're still in the second stride
    memory.forceEnterLowMemoryMode();


    // Write 250 more rows (which still gets written to the second stride, but not enough to fill
    // it)
    for (int i = 0; i < 250; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i + 250)));
    }


    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);


    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're just starting the second stride
    memory.forceEnterLowMemoryMode();


    // Write 500 more rows (a portion of the second stride)
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);


    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1001; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i % 1000)));
    }


    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're just after starting the second stride
    memory.forceEnterLowMemoryMode();


    // Write 499 more rows (a portion of the second stride)
    for (int i = 1; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 10000, memory);


    // Write 500 rows, they wil be directly encoded
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Flush the first stripe
    memory.forceFlushStripe();


    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }


    // Force the writer to enter low memory mode
    memory.forceEnterLowMemoryMode();


    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new StringStruct(Integer.toString(i + 500)));
    }


    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

    MemoryManagerWithForceFlush memory = new MemoryManagerWithForceFlush(conf);
    ReaderWriterProfiler.setProfilerOptions(conf);
    OrcConf.setIntVar(conf, OrcConf.ConfVars.HIVE_ORC_ENTROPY_STRING_THRESHOLD, 1);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    writer.addRow(new StringStruct(""));
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;

View Full Code Here

    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_DICTIONARY_SORT_KEYS, true);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    // Write a stripe which is not dictionary encoded
    for (int i = 0; i < 2000; i++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }
    memory.forceFlushStripe();
    // Write another stripe (doesn't matter what)
    for (int i = 0; i < 2000; i++) {
      writer.addRow(new StringStruct(Integer.toString(i)));
    }
    memory.forceFlushStripe();
    // Write a stripe which will be dictionary encoded
    // Note: it is important that this string is lexicographically after the string in the next
    // index stride.  This way, if sorting by index strides is not working, this value will appear
    // after the next one, though it should appear before, yielding incorrect results.
    writer.addRow(new StringStruct("b"));
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    writer.addRow(new StringStruct("a"));
    for (int i = 0; i < 999; i++) {
      writer.addRow(new StringStruct("123"));
    }
    memory.forceFlushStripe();
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of com.facebook.hive.orc.OrcTestUtils.StringStruct

com.facebook.hive.orc.TestOrcFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.