Examples of org.apache.hadoop.zebra.io.TableScanner

org.apache.hadoop.zebra.io.TableScanner
Scanner interface allows application to scan a range of rows of a table. It is the caller's responsibility to close() the scanner after finishing using it. Scanner allows one to fetch the key without fetching the value to allow underlying implementation to instantiating tuple objects (Row) lazily.

  public void testReadRecordOfMap1() throws IOException, ParseException {
    String projection1 = new String("r2.r3.f4#{a|b}, r2.r3.f3");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection1);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(1, ((Map) (RowValue.get(0))).get("a"));
    Assert.assertEquals(2, ((Map) RowValue.get(0)).get("b"));
    Assert.assertEquals(1.3, RowValue.get(1));


    scanner.advance();
    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("a"));
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("b"));
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("x"));
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("y"));
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("c"));

View Full Code Here

  public void testReadRecordOfRecord2() throws IOException, ParseException {
    String projection2 = new String("r1.f1, r2.r3.f3");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection2);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(1, RowValue.get(0));
    Assert.assertEquals(1.3, RowValue.get(1));


    scanner.advance();


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(2, RowValue.get(0));
    Assert.assertEquals(2.3, RowValue.get(1));
    reader.close();
  }

View Full Code Here

  public void testReadRecordOfRecord3() throws IOException, ParseException {
    String projection3 = new String("r1, r2");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection3);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);


    Assert.assertEquals(1, ((Tuple) RowValue.get(0)).get(0));
    Assert.assertEquals(1001L, ((Tuple) RowValue.get(0)).get(1));
    Tuple r2 = (Tuple) RowValue.get(1);
    Tuple r3 = (Tuple) r2.get(0);
    Map<String, Integer> f4 = (Map<String, Integer>) r3.get(1);
    Integer tmp = 1;
    Assert.assertEquals(tmp, f4.get("a"));
    tmp = 2;
    Assert.assertEquals(tmp, f4.get("b"));
    tmp = 3;
    Assert.assertEquals(tmp, f4.get("c"));
    Assert.assertEquals(1.3, ((Tuple) ((Tuple) RowValue.get(1)).get(0)).get(0));


    scanner.advance();


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(2, ((Tuple) RowValue.get(0)).get(0));
    Assert.assertEquals(1002L, ((Tuple) RowValue.get(0)).get(1));
    r2 = (Tuple) RowValue.get(1);
    r3 = (Tuple) r2.get(0);
    f4 = (Map<String, Integer>) r3.get(1);

View Full Code Here

  public void testReadRecordOfRecord4() throws IOException, ParseException {
    String projection3 = new String("r2.r3.f4");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection3);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);


    Map<String, Integer> f4 = (Map<String, Integer>) RowValue.get(0);
    Integer tmp = 1;
    Assert.assertEquals(tmp, f4.get("a"));
    tmp = 2;
    Assert.assertEquals(tmp, f4.get("b"));
    tmp = 3;
    Assert.assertEquals(tmp, f4.get("c"));
    Assert.assertEquals(null, f4.get("x"));


    scanner.advance();


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);


    f4 = (Map<String, Integer>) RowValue.get(0);
    tmp = 11;
    Assert.assertEquals(tmp, f4.get("x"));
    tmp = 12;

View Full Code Here

  public void testReadNegative1() throws IOException, ParseException {
    String projection4 = new String("r3");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection4);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(null, RowValue.get(0));
    reader.close();


  }

View Full Code Here

    TableInputFormat.requireSortedTable(conf1, null);
    TableInputFormat tif = new TableInputFormat();
 
    SortedTableSplit split = (SortedTableSplit) tif.getSplits(conf1, 1)[0];
    
    TableScanner scanner = reader.getScanner(split.getBegin(), split.getEnd(), true);
    BytesWritable key = new BytesWritable();
    Tuple rowValue = TypesUtils.createTuple(scanner.getSchema());


    Object lastVal = null;
    int numbCols1 = 0;
    long numbVerifiedRows = 0;


    while (!scanner.atEnd()) {
      ++numbRows;
      scanner.getKey(key);


      scanner.getValue(rowValue);


      // Verify every nth row
      if ((numbRows % rowMod) == 0) {
        ++numbVerifiedRows;
        numbCols1 = rowValue.size();
        if (numbCols != 0)
          Assert.assertEquals(
              "Verify failed - Table1 has wrong number of expected columns "
                  + "\n row numberrr : " + numbRows
                  + "\n expected column size : " + numbCols
                  + "\n actual columns size  : " + numbCols1, numbCols,
              numbCols1);


        Object newVal = rowValue.get(sortCol);


        // Verify sort key is in sorted order
        Assert.assertTrue("Verify failed - Table1 sort comparison error "
            + "\n row number : " + numbRows + "\n sort column : " + sortCol
            + "\n sort column last value    : " + lastVal
            + "\n sort column current value : " + newVal, compareTo(newVal,
            lastVal) >= 0);


        lastVal = newVal; // save last compare value


        //
        // Optionally verify data
        //
       
        if (verifyDataColName != null && verifyDataColName.equals("long1")) {
          Object newValLong1 = rowValue.get(sortCol);
          if (numbRows < 2000){
            System.out.println("Row : "+ (numbRows-1) +" long1 value : "+newValLong1.toString());
          }
          Assert.assertEquals(
              "Verify failed - Union table data verification error for column name : "
                  + verifyDataColName + "\n row number : " + (numbRows-1)
                  + "\n expected value : " + (numbRows-1 + 4) / 4 + // long1 will start with value 1
                  "\n actual value   : " + newValLong1, (numbRows-1 + 4) / 4,
              newValLong1);


        }


        scanner.advance();
      }
      


    }

View Full Code Here

     */
    String projection = new String("m1#{a}");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    System.out.println("read1 : " + RowValue.toString());
    Assert.assertEquals("{a=A}", RowValue.get(0).toString());


    scanner.advance();
    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    System.out.println(RowValue.get(0).toString());
    Assert.assertEquals("{a=A2}", RowValue.get(0).toString());


    reader.close();
  }

View Full Code Here


    TableInputFormat.requireSortedTable(conf1, null);
    TableInputFormat tif = new TableInputFormat();


  
    TableScanner scanner = reader.getScanner(null, null, true);
    BytesWritable key = new BytesWritable();
    Tuple rowValue = TypesUtils.createTuple(scanner.getSchema());


    while (!scanner.atEnd()) {
      ++numbRows;
      scanner.getKey(key);
      scanner.advance();
    }
    System.out.println("\nTable Path : " + pathTable1);
    System.out.println("Table Row number : " + numbRows);
  }

View Full Code Here

     */
    String projection2 = new String("m1#{b}, m2#{x|z}");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection2);
    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    System.out.println("map of map: " + RowValue.toString());
    // map of map: ([b#B],[z#,x#{m311=311, m321=321, m331=331}])
    Assert.assertEquals("B", ((Map) RowValue.get(0)).get("b"));
    Assert.assertEquals(321, ((Map) ((Map) RowValue.get(1)).get("x"))
        .get("m321"));
    Assert.assertEquals(311, ((Map) ((Map) RowValue.get(1)).get("x"))
        .get("m311"));
    Assert.assertEquals(331, ((Map) ((Map) RowValue.get(1)).get("x"))
        .get("m331"));
    Assert.assertEquals(null, ((Map) ((Map) RowValue.get(1)).get("x"))
        .get("m341"));
    Assert.assertEquals(null, ((Map) ((Map) RowValue.get(1)).get("z")));
    Assert.assertEquals(null, ((Map) ((Map) RowValue.get(0)).get("a")));
    Assert.assertEquals(null, ((Map) ((Map) RowValue.get(0)).get("c")));


    System.out.println("rowValue.get)1): " + RowValue.get(1).toString());
    // rowValue.get)1): {z=null, x={m311=311, m321=321, m331=331}}


    scanner.advance();


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(null, ((Map) RowValue.get(0)).get("b"));
    Assert.assertEquals(null, ((Map) ((Map) RowValue.get(1)).get("x")));
    Assert.assertEquals(323, ((Map) ((Map) RowValue.get(1)).get("z"))
        .get("m323"));
    Assert.assertEquals(322, ((Map) ((Map) RowValue.get(1)).get("z"))

View Full Code Here

    String projection = new String("m5");
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(projection);


    List<RangeSplit> splits = reader.rangeSplit(1);
    TableScanner scanner = reader.getScanner(splits.get(0), true);
    BytesWritable key = new BytesWritable();
    Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());


    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(false, RowValue.isNull());
    Assert.assertEquals(null, RowValue.get(0));
    Assert.assertEquals(1, RowValue.size());


    scanner.advance();
    scanner.getKey(key);
    Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
    scanner.getValue(RowValue);
    Assert.assertEquals(false, RowValue.isNull());
    Assert.assertEquals(null, RowValue.get(0));
    Assert.assertEquals(1, RowValue.size());
    reader.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.zebra.io.TableScanner

org.apache.hadoop.zebra.io.TestBasicTableMapSplits

org.apache.hadoop.zebra.io.TestBasicTableProjections

org.apache.hadoop.zebra.io.TestBasicTableSplits

org.apache.hadoop.zebra.io.TestCollection

org.apache.hadoop.zebra.io.TestColumnGroup

org.apache.hadoop.zebra.io.TestColumnGroupName1

org.apache.hadoop.zebra.io.TestColumnGroupProjections

org.apache.hadoop.zebra.io.TestColumnGroupReaders

org.apache.hadoop.zebra.io.TestColumnGroupSplits

org.apache.hadoop.zebra.io.TestColumnGroupWithWorkPath

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.