Examples of org.apache.accumulo.core.client.BatchScanner

org.apache.accumulo.core.client.BatchScanner
Implementations of BatchScanner support efficient lookups of many ranges in accumulo. Use this when looking up lots of ranges and you expect each range to contain a small amount of data. Also only use this when you do not care about the returned data being in sorted order. If you want to lookup a few ranges and expect those ranges to contain a lot of data, then use the Scanner instead. Also, the Scanner will return data in sorted order, this will not.

    
    bw.addMutation(m);
    
    bw.close();
    
    BatchScanner bs = getConnector().createBatchScanner("timeout", Constants.NO_AUTHS, 2);
    bs.setTimeout(1, TimeUnit.SECONDS);
    bs.setRanges(Collections.singletonList(new Range()));
    
    // should not timeout
    for (Entry<Key,Value> entry : bs) {
      entry.getKey();
    }
    
    IteratorSetting iterSetting = new IteratorSetting(100, SlowIterator.class);
    iterSetting.addOption("sleepTime", 2000 + "");
    getConnector().tableOperations().attachIterator("timeout", iterSetting);
    UtilWaitThread.sleep(250);


    try {
      for (Entry<Key,Value> entry : bs) {
        entry.getKey();
      }
      throw new Exception("batch scanner did not time out");
    } catch (TimedOutException toe) {
      // toe.printStackTrace();
    }


    bs.close();
  }

View Full Code Here

    
    if (!caught)
      throw new Exception("Scan did not fail");
    
    // try to batch scan the table
    BatchScanner bs = getConnector().createBatchScanner("tt", Constants.NO_AUTHS, 2);
    bs.setRanges(Collections.singleton(new Range()));
    
    caught = false;
    try {
      for (Entry<Key,Value> entry : bs) {
        entry.getKey();

View Full Code Here

    System.out.printf("Multiple thread scan time %6.2f %n", time2 / 1000.0);
    
  }
  
  private long batchScan(List<Range> ranges, int threads) throws Exception {
    BatchScanner bs = getConnector().createBatchScanner("test_ingest", TestIngest.AUTHS, threads);
    
    bs.setRanges(ranges);
    
    int count = 0;
    
    long t1 = System.currentTimeMillis();
    
    byte rval[] = new byte[50];
    Random random = new Random();
    
    for (Entry<Key,Value> entry : bs) {
      count++;
      int row = VerifyIngest.getRow(entry.getKey());
      int col = VerifyIngest.getCol(entry.getKey());
      
      if (row < 0 || row >= NUM_TO_INGEST) {
        throw new Exception("unexcepted row " + row);
      }
      
      rval = TestIngest.genRandomValue(random, rval, 2, row, col);
      
      if (entry.getValue().compareTo(rval) != 0) {
        throw new Exception("unexcepted value row=" + row + " col=" + col);
      }
    }
    
    long t2 = System.currentTimeMillis();
    
    bs.close();
    
    if (count != NUM_TO_INGEST) {
      throw new Exception("Batch Scan did not return expected number of values " + count);
    }

View Full Code Here

      Mutation m = new Mutation(row);
      m.put("cf", "cq", "");
      bw.addMutation(m);
    }
    bw.flush();
    BatchScanner bs = conn.createBatchScanner("test", Constants.NO_AUTHS, 1);
    IteratorSetting cfg = new IteratorSetting(100, TransformIterator.class);
    bs.addScanIterator(cfg);
    bs.setRanges(Collections.singletonList(new Range("A", "Z")));
    StringBuilder sb = new StringBuilder();
    String comma = "";
    for (Entry<Key,Value>  entry : bs) {
      sb.append(comma);
      sb.append(entry.getKey().getRow());

View Full Code Here

        auth = connector.securityOperations().getUserAuthorizations(connector.whoami());
      }
      if (opts != null && opts.threads > 0)
        threads = opts.threads;
      
      BatchScanner scanner = connector.createBatchScanner(tableName, auth, threads);
      
      if (opts != null) {
        if (opts.iterators != null) {
          for (org.apache.accumulo.proxy.thrift.IteratorSetting iter : opts.iterators) {
            IteratorSetting is = new IteratorSetting(iter.getPriority(), iter.getName(), iter.getIteratorClass(), iter.getProperties());
            scanner.addScanIterator(is);
          }
        }
        
        ArrayList<Range> ranges = new ArrayList<Range>();
        
        if (opts.ranges == null) {
          ranges.add(new Range());
        } else {
          for (org.apache.accumulo.proxy.thrift.Range range : opts.ranges) {
            Range aRange = new Range(range.getStart() == null ? null : Util.fromThrift(range.getStart()), true, range.getStop() == null ? null
                : Util.fromThrift(range.getStop()), false);
            ranges.add(aRange);
          }
        }
        scanner.setRanges(ranges);
        
        if (opts.columns != null) {
          for (ScanColumn col : opts.columns) {
            if (col.isSetColQualifier())
              scanner.fetchColumn(ByteBufferUtil.toText(col.colFamily), ByteBufferUtil.toText(col.colQualifier));
            else
              scanner.fetchColumnFamily(ByteBufferUtil.toText(col.colFamily));
          }
        }
      }
      
      UUID uuid = UUID.randomUUID();
      
      ScannerPlusIterator spi = new ScannerPlusIterator();
      spi.scanner = scanner;
      spi.iterator = scanner.iterator();
      scannerCache.put(uuid, spi);
      return uuid.toString();
    } catch (Exception e) {
      handleExceptionTNF(e);
      return null;

View Full Code Here

    Opts opts = new Opts();
    BatchScannerOpts bsOpts = new BatchScannerOpts();
    opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
    
    Connector connector = opts.getConnector();
    BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
    batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
    
    Random r;
    if (opts.seed == null)
      r = new Random();
    else
      r = new Random(opts.seed);
    
    // do one cold
    doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
    
    System.gc();
    System.gc();
    System.gc();
    
    if (opts.seed == null)
      r = new Random();
    else
      r = new Random(opts.seed);
    
    // do one hot (connections already established, metadata table cached)
    doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
    
    batchReader.close();
  }

View Full Code Here

  static class CloseScanner implements RemovalListener<UUID,ScannerPlusIterator> {
    @Override
    public void onRemoval(RemovalNotification<UUID,ScannerPlusIterator> notification) {
      final ScannerBase base = notification.getValue().scanner;
      if (base instanceof BatchScanner) {
        final BatchScanner scanner = (BatchScanner) base;
        scanner.close();
      }
    }

View Full Code Here

      Mutation m = new Mutation(asText(r));
      m.put(asText(random.nextInt()), asText(random.nextInt()), new Value(Integer.toHexString(r).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    BatchScanner s = c.createBatchScanner("test", Constants.NO_AUTHS, 2);
    s.setRanges(Collections.singletonList(new Range()));
    Key key = null;
    int count = 0;
    for (Entry<Key,Value> entry : s) {
      if (key != null)
        assertTrue(key.compareTo(entry.getKey()) < 0);

View Full Code Here

    m1.put("cat", "doc03", vis3, "");
    
    bw.addMutation(m1);
    bw.close();
    
    BatchScanner bs = connector.createBatchScanner("shard_table", authorizations, 1);
    
    bs.addScanIterator(new IteratorSetting(21, ColVisReversingKeyTransformingIterator.class));
    IteratorSetting iicfg = new IteratorSetting(22, IntersectingIterator.class);
    IntersectingIterator.setColumnFamilies(iicfg, new Text[] {new Text("foo"), new Text("dog"), new Text("cat")});
    bs.addScanIterator(iicfg);
    bs.setRanges(Collections.singleton(new Range()));
    
    Iterator<Entry<Key,Value>> iter = bs.iterator();
    assertTrue(iter.hasNext());
    Key docKey = iter.next().getKey();
    assertEquals("shard001", docKey.getRowData().toString());
    assertEquals("doc02", docKey.getColumnQualifierData().toString());
    assertFalse(iter.hasNext());
    
    bs.close();
  }

View Full Code Here

      
      undefs.add(new UndefinedNode(undef, ref));
    }
    
    Connector conn = opts.getConnector();
    BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads);
    bscanner.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
    List<Range> refs = new ArrayList<Range>();
    
    for (UndefinedNode undefinedNode : undefs)
      refs.add(new Range(new Text(undefinedNode.ref)));
    
    bscanner.setRanges(refs);
    
    HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>();
    
    for (Entry<Key,Value> entry : bscanner) {
      String ref = entry.getKey().getRow().toString();
      List<String> vals = refInfo.get(ref);
      if (vals == null) {
        vals = new ArrayList<String>();
        refInfo.put(ref, vals);
      }
      
      vals.add(entry.getValue().toString());
    }
    
    bscanner.close();
    
    IngestInfo ingestInfo = new IngestInfo(opts.logDir);
    TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(conn.getInstance(), opts.getTableName()), opts.logDir);
    
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchScanner

mil.nga.giat.geowave.accumulo.metadata.AbstractAccumuloPersistence

mil.nga.giat.geowave.vector.query.DistributedRenderQuery

org.apache.accumulo.core.client.mock.MockConnectorTest

org.apache.accumulo.core.client.mock.MockTableOperationsTest

org.apache.accumulo.core.client.mock.TestBatchScanner821

org.apache.accumulo.core.iterators.user.IntersectingIteratorTest

org.apache.accumulo.core.iterators.user.TransformingIteratorTest

org.apache.accumulo.core.util.shell.commands.GrepCommand

org.apache.accumulo.core.util.shell.Shell$GrepCommand

org.apache.accumulo.examples.client.RandomBatchScanner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.