Package com.facebook.hive.orc.lazy

Examples of com.facebook.hive.orc.lazy.OrcLazyStruct

933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    rand = new Random(12);
    OrcLazyStruct row = null;
    for(int i=0; i < 10000; ++i) {
      assertEquals(true, rows.hasNext());
      row = (OrcLazyStruct) rows.next(row);
      assertEquals(rand.nextInt(), ((IntWritable) ((OrcLazyInt) ((OrcStruct) row.materialize()).getFieldValue(0)).materialize()).get());
      assertEquals(Integer.toHexString(rand.nextInt()),
          ((OrcLazyString) ((OrcStruct) row.materialize()).getFieldValue(1)).materialize().toString());
    }
    assertEquals(false, rows.hasNext());
    rows.close();
  }
View Full Code Here
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
    StripeInformation stripe = reader.getStripes().iterator().next();
    assertEquals(true, stripe.getDataLength() != 0);
    assertEquals(0, stripe.getIndexLength());
    RecordReader rows = reader.rows(null);
    rand = new Random(24);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    for(int i=0; i < 10000; ++i) {
      int intVal = rand.nextInt();
      String strVal = Integer.toBinaryString(rand.nextInt());
      for(int j=0; j < 5; ++j) {
        assertEquals(true, rows.hasNext());
        lazyRow = (OrcLazyStruct) rows.next(lazyRow);
        row = (OrcStruct) lazyRow.materialize();
        assertEquals(intVal, ((IntWritable) ((OrcLazyInt) row.getFieldValue(0)).materialize()).get());
        assertEquals(strVal, ((OrcLazyString) row.getFieldValue(1)).materialize().toString());
      }
    }
    assertEquals(false, rows.hasNext());
View Full Code Here
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
    ReaderWriterProfiler.setProfilerOptions(conf);
    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_EAGER_HDFS_READ, !lazyHdfsReads);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    assertEquals(COUNT, reader.getNumberOfRows());
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    for(int i=COUNT-1; i >= 0; --i) {
      rows.seekToRow(i);
      lazyRow = (OrcLazyStruct) rows.next(lazyRow);
      row = (OrcStruct) lazyRow.materialize();
      compareRows(row, inputs, i, NumberOfNulls.NONE, testPrimitives);
    }
    rows.close();
  }
View Full Code Here
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
    ReaderWriterProfiler.setProfilerOptions(conf);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    assertEquals(COUNT, reader.getNumberOfRows());
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    for(int i = 0; i < COUNT / n; i++) {
      rows.seekToRow(i * n);
      lazyRow = (OrcLazyStruct) rows.next(lazyRow);
      row = (OrcStruct) lazyRow.materialize();
      if (withoutNextIsNull) {
        compareRowsWithoutNextIsNull(row, inputs, i * n, numNulls, true);
      } else {
        compareRows(row, inputs, i * n, numNulls, true);
      }
View Full Code Here
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
      writer.addRow(new StringStruct("123"));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals("", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
    for (int i =0; i < 999; i++) {
      rows.next(lazyRow);
      assertEquals("123", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
    }
View Full Code Here
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
    }
    writer.forceFlushStripe();
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    for (int i =0; i < 4000; i++) {
      assertEquals(Integer.toString(i % 2000), ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
      rows.next(lazyRow);
    }
    assertEquals("b", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
View Full Code Here
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
    }
    writer.close();

    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    // Skip the one row in the stride dictionary in the first stride ("a")
    rows.next(lazyRow);
    // Read the rest of the values in the stride
    for (int i =0; i < 999; i++) {
      assertEquals("123", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
View Full Code Here
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
      writer.addRow(new StringStruct("123"));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals("a", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
    rows.next(lazyRow);
    assertEquals("b", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
    rows.next(lazyRow);
    assertEquals("c", ((OrcLazyString) row.getFieldValue(0)).materialize().toString());
View Full Code Here
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
      writer.addRow(new IntStruct(123));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
    OrcStruct row = null;
    lazyRow = (OrcLazyStruct) rows.next(lazyRow);
    row = (OrcStruct) lazyRow.materialize();
    assertEquals(1, ((IntWritable) ((OrcLazyInt) row.getFieldValue(0)).materialize()).get());
    rows.next(lazyRow);
    assertEquals(2, ((IntWritable) ((OrcLazyInt) row.getFieldValue(0)).materialize()).get());
    rows.next(lazyRow);
    assertEquals(3, ((IntWritable) ((OrcLazyInt) row.getFieldValue(0)).materialize()).get());
View Full Code Here
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
    // Prepare to read back the data
    ReaderWriterProfiler.setProfilerOptions(conf);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = (OrcLazyStruct) rows.next(null);
    OrcStruct row = (OrcStruct) lazyRow.materialize();
    OrcLazyList list = ((OrcLazyList) row.getFieldValue(1));
    LazyTreeReader lazyReader = list.getLazyTreeReader();

    Object prev = lazyReader.get(numNonNulls - 1, null);
View Full Code Here
TOP

Related Classes of com.facebook.hive.orc.lazy.OrcLazyStruct

  • com.facebook.hive.orc.RecordReaderImpl
  • com.facebook.hive.orc.TestMapTreeWriter
  • com.facebook.hive.orc.TestObjectInspector
  • com.facebook.hive.orc.TestOrcFile
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.