Examples of org.apache.hadoop.hbase.regionserver.InternalScanner

Package org.apache.hadoop.hbase.regionserver

Examples of org.apache.hadoop.hbase.regionserver.InternalScanner

org.apache.hadoop.hbase.regionserver.InternalScanner
Internal scanners differ from client-side scanners in that they operate on HStoreKeys and byte[] instead of RowResults. This is because they are actually close to how the data is physically stored, and therefore it is more convenient to interact with them that way. It is also much easier to merge the results across SortedMaps than RowResults.
Additionally, we need to be able to determine if the scanner is doing wildcard column matches (when only a column family is specified or if a column regex is specified) or if multiple members of the same column family were specified. If so, we need to ignore the timestamp to ensure that we get all the family members, as they may have been last updated at different times.

    // do a full scan of _acl_ table


    Scan scan = new Scan();
    scan.addFamily(ACL_LIST_FAMILY);


    InternalScanner iScanner = null;
    try {
      iScanner = aclRegion.getScanner(scan);


      while (true) {
        List<Cell> row = new ArrayList<Cell>();


        boolean hasNext = iScanner.next(row);
        ListMultimap<String,TablePermission> perms = ArrayListMultimap.create();
        byte[] entry = null;
        for (Cell kv : row) {
          if (entry == null) {
            entry = CellUtil.cloneRow(kv);
          }
          Pair<String,TablePermission> permissionsOfUserOnTable =
              parsePermissionRecord(entry, kv);
          if (permissionsOfUserOnTable != null) {
            String username = permissionsOfUserOnTable.getFirst();
            TablePermission permissions = permissionsOfUserOnTable.getSecond();
            perms.put(username, permissions);
          }
        }
        if (entry != null) {
          allPerms.put(entry, perms);
        }
        if (!hasNext) {
          break;
        }
      }
    } finally {
      if (iScanner != null) {
        iScanner.close();
      }
    }


    return allPerms;
  }

View Full Code Here

    long lastFlush;


    @Override
    public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
        Store store, final InternalScanner scanner) {
      return new InternalScanner() {
        @Override
        public boolean next(List<KeyValue> results) throws IOException {
          return next(results, -1);
        }

View Full Code Here


    Scan s = new Scan();
    WhileMatchFilter filter = new WhileMatchFilter(new PageFilter(pageSize));
    s.setFilter(filter);


    InternalScanner scanner = this.region.getScanner(s);
    int scannerCounter = 0;
    while (true) {
      boolean isMoreResults = scanner.next(new ArrayList<KeyValue>());
      scannerCounter++;


      if (scannerCounter >= pageSize) {
        Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
      }

View Full Code Here

    Scan s = new Scan();
    String prefix = "testRowOne";
    WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(Bytes.toBytes(prefix)));
    s.setFilter(filter);


    InternalScanner scanner = this.region.getScanner(s);
    while (true) {
      ArrayList<KeyValue> values = new ArrayList<KeyValue>();
      boolean isMoreResults = scanner.next(values);
      if (!isMoreResults || !Bytes.toString(values.get(0).getRow()).startsWith(prefix)) {
        Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
      }
      if (!isMoreResults) {
        break;

View Full Code Here

    WhileMatchFilter filter = new WhileMatchFilter(
        new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, Bytes.toBytes("foo"))
    );
    s.setFilter(filter);


    InternalScanner scanner = this.region.getScanner(s);
    while (true) {
      ArrayList<KeyValue> values = new ArrayList<KeyValue>();
      boolean isMoreResults = scanner.next(values);
      Assert.assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
      if (!isMoreResults) {
        break;
      }
    }

View Full Code Here

    // combine these two with OR in a FilterList
    FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf);


    Scan s1 = new Scan();
    s1.setFilter(filterList);
    InternalScanner scanner = testRegion.getScanner(s1);
    List<KeyValue> results = new ArrayList<KeyValue>();
    int resultCount = 0;
    while(scanner.next(results)) {
      resultCount++;
      byte[] row = results.get(0).getRow();
      LOG.debug("Found row: " + Bytes.toStringBinary(row));
      Assert.assertTrue(Bytes.equals(row, Bytes.toBytes("brow"))
          || Bytes.equals(row, Bytes.toBytes("crow")));
      results.clear();
    }
    Assert.assertEquals(2, resultCount);
    scanner.close();


    HLog hlog = testRegion.getLog();
    testRegion.close();
    hlog.closeAndDelete();
  }

View Full Code Here

    // 1. got rows > "row4"
    Filter rowFilter = new RowFilter(CompareOp.GREATER, new BinaryComparator(
        Bytes.toBytes("row4")));
    Scan s1 = new Scan();
    s1.setFilter(rowFilter);
    InternalScanner scanner = testRegion.getScanner(s1);
    List<KeyValue> results = new ArrayList<KeyValue>();
    int i = 5;
    for (boolean done = true; done; i++) {
      done = scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    // 2. got rows <= "row4" and S=
    FilterList subFilterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    Filter subFilter1 = new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(
        Bytes.toBytes("row4")));
    subFilterList.addFilter(subFilter1);
    Filter subFilter2 = new SingleColumnValueFilter(FAMILIES[0], columnStatus, CompareOp.EQUAL,
        Bytes.toBytes(0));
    subFilterList.addFilter(subFilter2);
    s1 = new Scan();
    s1.setFilter(subFilterList);
    scanner = testRegion.getScanner(s1);
    results = new ArrayList<KeyValue>();
    for (i=0; i<=4; i+=2) {
      scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    Assert.assertFalse(scanner.next(results));
    // 3. let's begin to verify nested filter list
    // 3.1 add rowFilter, then add subFilterList
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    filterList.addFilter(rowFilter);
    filterList.addFilter(subFilterList);
    s1 = new Scan();
    s1.setFilter(filterList);
    scanner = testRegion.getScanner(s1);
    results = new ArrayList<KeyValue>();
    for (i=0; i<=4; i+=2) {
      scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    for (i=5; i<=9; i++) {
      scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    Assert.assertFalse(scanner.next(results));
    // 3.2 MAGIC here! add subFilterList first, then add rowFilter
    filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    filterList.addFilter(subFilterList);
    filterList.addFilter(rowFilter);
    s1 = new Scan();
    s1.setFilter(filterList);
    scanner = testRegion.getScanner(s1);
    results = new ArrayList<KeyValue>();
    for (i=0; i<=4; i+=2) {
      scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    for (i=5; i<=9; i++) {
      scanner.next(results);
      Assert.assertTrue(Bytes.equals(results.get(0).getRow(), Bytes.toBytes("row" + i)));
      Assert.assertEquals(Bytes.toInt(results.get(0).getValue()), i%2);
      results.clear();
    }
    Assert.assertFalse(scanner.next(results));
    HLog hlog = testRegion.getLog();
    testRegion.close();
    hlog.closeAndDelete();
  }

View Full Code Here


  }


  private void verifyScan(Scan s, long expectedRows, long expectedKeys)
  throws IOException {
    InternalScanner scanner = this.region.getScanner(s);
    List<KeyValue> results = new ArrayList<KeyValue>();
    int i = 0;
    for (boolean done = true; done; i++) {
      done = scanner.next(results);
      Arrays.sort(results.toArray(new KeyValue[results.size()]),
          KeyValue.COMPARATOR);
      LOG.info("counter=" + i + ", " + results);
      if (results.isEmpty()) break;
      assertTrue("Scanned too many rows! Only expected " + expectedRows +

View Full Code Here

  }


  private void verifyScanNoEarlyOut(Scan s, long expectedRows,
      long expectedKeys)
  throws IOException {
    InternalScanner scanner = this.region.getScanner(s);
    List<KeyValue> results = new ArrayList<KeyValue>();
    int i = 0;
    for (boolean done = true; done; i++) {
      done = scanner.next(results);
      Arrays.sort(results.toArray(new KeyValue[results.size()]),
          KeyValue.COMPARATOR);
      LOG.info("counter=" + i + ", " + results);
      if(results.isEmpty()) break;
      assertTrue("Scanned too many rows! Only expected " + expectedRows +

View Full Code Here

        " rows", expectedRows, i);
  }


  private void verifyScanFull(Scan s, KeyValue [] kvs)
  throws IOException {
    InternalScanner scanner = this.region.getScanner(s);
    List<KeyValue> results = new ArrayList<KeyValue>();
    int row = 0;
    int idx = 0;
    for (boolean done = true; done; row++) {
      done = scanner.next(results);
      Arrays.sort(results.toArray(new KeyValue[results.size()]),
          KeyValue.COMPARATOR);
      if(results.isEmpty()) break;
      assertTrue("Scanned too many keys! Only expected " + kvs.length +
          " total but already scanned " + (results.size() + idx) +

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.InternalScanner

co.nubetech.crux.server.aggregate.GroupingAggregationImpl

com.alibaba.wasp.coprocessor.WaspAggregateImplementation

com.sematext.hbase.hut.cp.HutReadEndpoint

coprocessor.RowCountEndpoint

HBaseIA.TwitBase.coprocessors.RelationCountImpl

org.apache.hadoop.hbase.coprocessor.AggregateImplementation

org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint

org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpointNullResponse

org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpointWithErrors

org.apache.hadoop.hbase.coprocessor.example.RowCountEndpoint

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.