Examples of org.apache.accumulo.core.client.BatchScanner

org.apache.accumulo.core.client.BatchScanner
Implementations of BatchScanner support efficient lookups of many ranges in accumulo. Use this when looking up lots of ranges and you expect each range to contain a small amount of data. Also only use this when you do not care about the returned data being in sorted order. If you want to lookup a few ranges and expect those ranges to contain a lot of data, then use the Scanner instead. Also, the Scanner will return data in sorted order, this will not.

    m.put("rvy", "5000000000000000", empty);
    m.put("15qh", "5000000000000000", empty);
    bw.addMutation(m);
    bw.close();
    
    BatchScanner bs = connector.createBatchScanner("index", Constants.NO_AUTHS, 10);
    IteratorSetting ii = new IteratorSetting(20, IntersectingIterator.class);
    IntersectingIterator.setColumnFamilies(ii, new Text[] {new Text("rvy"), new Text("15qh")});
    bs.addScanIterator(ii);
    bs.setRanges(Collections.singleton(new Range()));
    Iterator<Entry<Key,Value>> iterator = bs.iterator();
    Assert.assertTrue(iterator.hasNext());
    Entry<Key,Value> next = iterator.next();
    Key key = next.getKey();
    Assert.assertEquals(key.getColumnQualifier(), new Text("5000000000000000"));
    Assert.assertFalse(iterator.hasNext());

View Full Code Here

      undefs.add(new UndefinedNode(undef, ref));
    }


    ZooKeeperInstance zki = new ZooKeeperInstance(instanceName, zooKeepers);
    Connector conn = zki.getConnector(user, password.getBytes());
    BatchScanner bscanner = conn.createBatchScanner(table, Constants.NO_AUTHS, 20);


    List<Range> refs = new ArrayList<Range>();


    for (UndefinedNode undefinedNode : undefs)
      refs.add(new Range(new Text(undefinedNode.ref)));


    bscanner.setRanges(refs);


    HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>();


    for (Entry<Key,Value> entry : bscanner) {
      String ref = entry.getKey().getRow().toString();
      List<String> vals = refInfo.get(ref);
      if (vals == null) {
        vals = new ArrayList<String>();
        refInfo.put(ref, vals);
      }


      vals.add(entry.getValue().toString());
    }


    bscanner.close();


    IngestInfo ingestInfo = new IngestInfo(logDir);
    TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(zki, table), acuLogDir);


    SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

View Full Code Here

    int refCount = 0;
    
    try {
      // look for other tables that references this tables files
      Connector conn = instance.getConnector(SecurityConstants.getSystemCredentials());
      BatchScanner bs = conn.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
      try {
        bs.setRanges(Collections.singleton(Constants.NON_ROOT_METADATA_KEYSPACE));
        bs.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
        IteratorSetting cfg = new IteratorSetting(40, "grep", GrepIterator.class);
        GrepIterator.setTerm(cfg, "../" + tableId + "/");
        bs.addScanIterator(cfg);
        
        for (Entry<Key,Value> entry : bs) {
          if (entry.getKey().getColumnQualifier().toString().startsWith("../" + tableId + "/")) {
            refCount++;
          }
        }
      } finally {
        bs.close();
      }
      
    } catch (Exception e) {
      refCount = -1;
      log.error("Failed to scan !METADATA looking for references to deleted table " + tableId, e);

View Full Code Here

    Opts opts = new Opts();
    BatchScannerOpts bsOpts = new BatchScannerOpts();
    opts.parseArgs(RandomBatchScanner.class.getName(), args, bsOpts);
    
    Connector connector = opts.getConnector();
    BatchScanner batchReader = connector.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads);
    batchReader.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
    
    Random r;
    if (opts.seed == null)
      r = new Random();
    else
      r = new Random(opts.seed);
    
    // do one cold
    boolean status = doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
    
    System.gc();
    System.gc();
    System.gc();
    
    if (opts.seed == null)
      r = new Random();
    else
      r = new Random(opts.seed);
    
    // do one hot (connections already established, metadata table cached)
    status = status && doRandomQueries(opts.num, opts.min, opts.max, opts.size, r, batchReader);
    
    batchReader.close();
    if (!status) {
      System.exit(1);
    }
  }

View Full Code Here

    RandomAuths randomAuths = new RandomAuths(authsFile);
    Authorizations auths = randomAuths.getAuths(r);


    Connector conn = new ZooKeeperInstance(instanceName, zooKeepers).getConnector(user, password.getBytes());
    Scanner scanner = ContinuousUtil.createScanner(conn, table, auths);
    BatchScanner bs = conn.createBatchScanner(table, auths, numQueryThreads);


    while (true) {
      Set<Text> batch = getBatch(scanner, min, max, batchSize, r);
      
      List<Range> ranges = new ArrayList<Range>(batch.size());

View Full Code Here

    // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
    // logger.setLevel(Level.TRACE);
    
    ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
    Connector connector = instance.getConnector(user, pass);
    BatchScanner tsbr = connector.createBatchScanner(table, new Authorizations(auths.split(",")), numThreads);
    
    Random r;
    if (seed == null)
      r = new Random();
    else
      r = new Random(Long.parseLong(seed));
    
    // do one cold
    status = doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
    
    System.gc();
    System.gc();
    System.gc();
    
    if (seed == null)
      r = new Random();
    else
      r = new Random(Long.parseLong(seed));
    
    // do one hot (connections already established, metadata table cached)
    status = status && doRandomQueries(num, min, max, expectedValueSize, r, tsbr);
    
    tsbr.close();
    if (!status) {
      System.exit(1);
    }
  }

View Full Code Here

  static class CloseScanner implements RemovalListener<UUID,ScannerPlusIterator> {
    @Override
    public void onRemoval(RemovalNotification<UUID,ScannerPlusIterator> notification) {
      final ScannerBase base = notification.getValue().scanner;
      if (base instanceof BatchScanner) {
        final BatchScanner scanner = (BatchScanner) base;
        scanner.close();
      }
    }

View Full Code Here

      }
      
      ranges.add(range);
    }
    
    BatchScanner bs = getConnector().createBatchScanner(table, Constants.NO_AUTHS, 3);
    bs.setRanges(ranges);
    
    long t1 = System.currentTimeMillis();
    
    for (Entry<Key,Value> entry : bs) {
      long v = Long.parseLong(entry.getValue().toString());
      if (!expected.remove(v)) {
        throw new Exception("Got unexpected return " + entry.getKey() + " " + entry.getValue());
      }
    }
    
    long t2 = System.currentTimeMillis();
    
    if (expected.size() > 0) {
      throw new Exception("Did not get all expected values " + expected.size());
    }
    
    bs.close();
    
    return t2 - t1;
  }

View Full Code Here

      ranges.add(new Range(new Text(String.format("%08x", (i << 8) - 16))));
    }
    
    getConnector().tableOperations().create("t3");
    getConnector().tableOperations().addSplits("t3", splits);
    BatchScanner bs = getConnector().createBatchScanner("t3", Constants.NO_AUTHS, 3);
    bs.setRanges(ranges);
    count = 0;
    for (Entry<Key,Value> entry : bs) {
      if (entry != null)
        count++;
    }
    
    if (count != 0) {
      throw new Exception("Did not see expected number of entries, count = " + count);
    }
    
    bs.close();
    
  }

View Full Code Here

    Scanner scanner = getConnector().createScanner("foo", new Authorizations());
    
    setupIter(scanner);
    verify(scanner, 1, 999);
    
    BatchScanner bscanner = getConnector().createBatchScanner("foo", new Authorizations(), 3);
    bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
    
    setupIter(bscanner);
    verify(bscanner, 1, 999);
    
    ArrayList<Range> ranges = new ArrayList<Range>();
    ranges.add(new Range(new Text(String.format("%06d", 1))));
    ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
    ranges.add(new Range(new Text(String.format("%06d", 20))));
    ranges.add(new Range(new Text(String.format("%06d", 23))));
    ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
    ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
    ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
    
    HashSet<Integer> got = new HashSet<Integer>();
    HashSet<Integer> expected = new HashSet<Integer>();
    for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
      expected.add(i);
    }
    
    bscanner.setRanges(ranges);
    
    for (Entry<Key,Value> entry : bscanner) {
      got.add(Integer.parseInt(entry.getKey().getRow().toString()));
    }
    
    System.out.println("got : " + got);
    
    if (!got.equals(expected)) {
      throw new Exception(got + " != " + expected);
    }
    
    bscanner.close();
    
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchScanner

mil.nga.giat.geowave.accumulo.metadata.AbstractAccumuloPersistence

mil.nga.giat.geowave.vector.query.DistributedRenderQuery

org.apache.accumulo.core.client.mock.MockConnectorTest

org.apache.accumulo.core.client.mock.MockTableOperationsTest

org.apache.accumulo.core.client.mock.TestBatchScanner821

org.apache.accumulo.core.iterators.user.IntersectingIteratorTest

org.apache.accumulo.core.iterators.user.TransformingIteratorTest

org.apache.accumulo.core.util.shell.commands.GrepCommand

org.apache.accumulo.core.util.shell.Shell$GrepCommand

org.apache.accumulo.examples.client.RandomBatchScanner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.