Examples of org.apache.accumulo.core.client.BatchScanner

org.apache.accumulo.core.client.BatchScanner
Implementations of BatchScanner support efficient lookups of many ranges in accumulo. Use this when looking up lots of ranges and you expect each range to contain a small amount of data. Also only use this when you do not care about the returned data being in sorted order. If you want to lookup a few ranges and expect those ranges to contain a lot of data, then use the Scanner instead. Also, the Scanner will return data in sorted order, this will not.

        auth = connector.securityOperations().getUserAuthorizations(connector.whoami());
      }
      if (opts != null && opts.threads > 0)
        threads = opts.threads;
      
      BatchScanner scanner = connector.createBatchScanner(tableName, auth, threads);
      
      if (opts != null) {
        if (opts.iterators != null) {
          for (org.apache.accumulo.proxy.thrift.IteratorSetting iter : opts.iterators) {
            IteratorSetting is = new IteratorSetting(iter.getPriority(), iter.getName(), iter.getIteratorClass(), iter.getProperties());
            scanner.addScanIterator(is);
          }
        }
        
        ArrayList<Range> ranges = new ArrayList<Range>();
        
        if (opts.ranges == null) {
          ranges.add(new Range());
        } else {
          for (org.apache.accumulo.proxy.thrift.Range range : opts.ranges) {
            Range aRange = new Range(range.getStart() == null ? null : Util.fromThrift(range.getStart()), true, range.getStop() == null ? null
                : Util.fromThrift(range.getStop()), false);
            ranges.add(aRange);
          }
        }
        scanner.setRanges(ranges);
        
        if (opts.columns != null) {
          for (ScanColumn col : opts.columns) {
            if (col.isSetColQualifier())
              scanner.fetchColumn(ByteBufferUtil.toText(col.colFamily), ByteBufferUtil.toText(col.colQualifier));
            else
              scanner.fetchColumnFamily(ByteBufferUtil.toText(col.colFamily));
          }
        }
      }
      
      UUID uuid = UUID.randomUUID();
      
      ScannerPlusIterator spi = new ScannerPlusIterator();
      spi.scanner = scanner;
      spi.iterator = scanner.iterator();
      scannerCache.put(uuid, spi);
      return uuid.toString();
    } catch (Exception e) {
      handleExceptionTNF(e);
      return null;

View Full Code Here

  
  private void verify(ByteArraySet nss, String... expected) throws Exception {
    Scanner scanner = getConnector().createScanner("vt", new Authorizations(nss));
    verify(scanner.iterator(), expected);
    
    BatchScanner bs = getConnector().createBatchScanner("vt", new Authorizations(nss), 3);
    bs.setRanges(Collections.singleton(new Range()));
    verify(bs.iterator(), expected);
    bs.close();
  }

View Full Code Here

      
      undefs.add(new UndefinedNode(undef, ref));
    }
    
    Connector conn = opts.getConnector();
    BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads);
    bscanner.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
    List<Range> refs = new ArrayList<Range>();
    
    for (UndefinedNode undefinedNode : undefs)
      refs.add(new Range(new Text(undefinedNode.ref)));
    
    bscanner.setRanges(refs);
    
    HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>();
    
    for (Entry<Key,Value> entry : bscanner) {
      String ref = entry.getKey().getRow().toString();
      List<String> vals = refInfo.get(ref);
      if (vals == null) {
        vals = new ArrayList<String>();
        refInfo.put(ref, vals);
      }
      
      vals.add(entry.getValue().toString());
    }
    
    bscanner.close();
    
    IngestInfo ingestInfo = new IngestInfo(opts.logDir);
    TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(conn.getInstance(), opts.getTableName()), opts.logDir);
    
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

View Full Code Here

      w.close();
    }
  }
  
  protected void assertVersionable(Connector c, String tableName, int size) throws TableNotFoundException {
    BatchScanner s = c.createBatchScanner(tableName, Constants.NO_AUTHS, 1);
    s.setRanges(Collections.singleton(Range.exact("row1", "cf", "cq")));
    int count = 0;
    for (Map.Entry<Key,Value> e : s) {
      Assert.assertEquals("row1", e.getKey().getRow().toString());
      Assert.assertEquals("cf", e.getKey().getColumnFamily().toString());
      Assert.assertEquals("cq", e.getKey().getColumnQualifier().toString());
      count++;
      
    }
    Assert.assertEquals(size, count);
    s.close();
  }

View Full Code Here

    int numThreads = 20;
    if (cl.hasOption(numThreadsOpt.getOpt())) {
      numThreads = Integer.parseInt(cl.getOptionValue(numThreadsOpt.getOpt()));
    }
    final Authorizations auths = getAuths(cl, shellState);
    final BatchScanner scanner = shellState.getConnector().createBatchScanner(tableName, auths, numThreads);
    scanner.setRanges(Collections.singletonList(getRange(cl, interpeter)));
    
    scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS);
    
    for (int i = 0; i < cl.getArgs().length; i++) {
      setUpIterator(Integer.MAX_VALUE - cl.getArgs().length + i, "grep" + i, cl.getArgs()[i], scanner, cl);
    }
    try {
      // handle columns
      fetchColumns(cl, scanner, interpeter);
      
      // output the records
      printRecords(cl, shellState, scanner, formatter, printFile);
    } finally {
      scanner.close();
    }
    
    return 0;
  }

View Full Code Here

  static class CloseScanner implements RemovalListener<UUID,ScannerPlusIterator> {
    @Override
    public void onRemoval(RemovalNotification<UUID,ScannerPlusIterator> notification) {
      final ScannerBase base = notification.getValue().scanner;
      if (base instanceof BatchScanner) {
        final BatchScanner scanner = (BatchScanner) base;
        scanner.close();
      }
    }

View Full Code Here

        auth = connector.securityOperations().getUserAuthorizations(connector.whoami());
      }
      if (opts != null && opts.threads > 0)
        threads = opts.threads;
      
      BatchScanner scanner = connector.createBatchScanner(tableName, auth, threads);
      
      if (opts != null) {
        if (opts.iterators != null) {
          for (org.apache.accumulo.proxy.thrift.IteratorSetting iter : opts.iterators) {
            IteratorSetting is = new IteratorSetting(iter.getPriority(), iter.getName(), iter.getIteratorClass(), iter.getProperties());
            scanner.addScanIterator(is);
          }
        }
        
        ArrayList<Range> ranges = new ArrayList<Range>();
        
        if (opts.ranges == null) {
          ranges.add(new Range());
        } else {
          for (org.apache.accumulo.proxy.thrift.Range range : opts.ranges) {
            Range aRange = new Range(range.getStart() == null ? null : Util.fromThrift(range.getStart()), true, range.getStop() == null ? null
                : Util.fromThrift(range.getStop()), false);
            ranges.add(aRange);
          }
        }
        scanner.setRanges(ranges);
        
        if (opts.columns != null) {
          for (ScanColumn col : opts.columns) {
            if (col.isSetColQualifier())
              scanner.fetchColumn(ByteBufferUtil.toText(col.colFamily), ByteBufferUtil.toText(col.colQualifier));
            else
              scanner.fetchColumnFamily(ByteBufferUtil.toText(col.colFamily));
          }
        }
      }
      
      UUID uuid = UUID.randomUUID();
      
      ScannerPlusIterator spi = new ScannerPlusIterator();
      spi.scanner = scanner;
      spi.iterator = scanner.iterator();
      scannerCache.put(uuid, spi);
      return uuid.toString();
    } catch (Exception e) {
      handleExceptionTNF(e);
      return null;

View Full Code Here


    Connector conn = opts.getConnector();
    Scanner scanner = ContinuousUtil.createScanner(conn, opts.getTableName(), auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    
    BatchScanner bs = conn.createBatchScanner(opts.getTableName(), auths, bsOpts.scanThreads);
    bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);


    while (true) {
      Set<Text> batch = getBatch(scanner, opts.min, opts.max, scanOpts.scanBatchSize, r);
      List<Range> ranges = new ArrayList<Range>(batch.size());

View Full Code Here

    Scanner scanner = getConnector().createScanner("foo", new Authorizations());
    
    setupIter(scanner);
    verify(scanner, 1, 999);
    
    BatchScanner bscanner = getConnector().createBatchScanner("foo", new Authorizations(), 3);
    bscanner.setRanges(Collections.singleton(new Range((Key) null, null)));
    
    setupIter(bscanner);
    verify(bscanner, 1, 999);
    
    ArrayList<Range> ranges = new ArrayList<Range>();
    ranges.add(new Range(new Text(String.format("%06d", 1))));
    ranges.add(new Range(new Text(String.format("%06d", 6)), new Text(String.format("%06d", 16))));
    ranges.add(new Range(new Text(String.format("%06d", 20))));
    ranges.add(new Range(new Text(String.format("%06d", 23))));
    ranges.add(new Range(new Text(String.format("%06d", 56)), new Text(String.format("%06d", 61))));
    ranges.add(new Range(new Text(String.format("%06d", 501)), new Text(String.format("%06d", 504))));
    ranges.add(new Range(new Text(String.format("%06d", 998)), new Text(String.format("%06d", 1000))));
    
    HashSet<Integer> got = new HashSet<Integer>();
    HashSet<Integer> expected = new HashSet<Integer>();
    for (int i : new int[] {1, 7, 9, 11, 13, 15, 23, 57, 59, 61, 501, 503, 999}) {
      expected.add(i);
    }
    
    bscanner.setRanges(ranges);
    
    for (Entry<Key,Value> entry : bscanner) {
      got.add(Integer.parseInt(entry.getKey().getRow().toString()));
    }
    
    System.out.println("got : " + got);
    
    if (!got.equals(expected)) {
      throw new Exception(got + " != " + expected);
    }
    
    bscanner.close();
    
  }

View Full Code Here

      ranges.add(new Range(row));
    }
    
    // logger.setLevel(Level.TRACE);
    
    BatchScanner bs = getConnector().createBatchScanner("bss", Constants.NO_AUTHS, 4);
    
    HashMap<Text,Value> found = new HashMap<Text,Value>();
    
    for (int i = 0; i < 20; i++) {
      
      found.clear();
      
      long t1 = System.currentTimeMillis();
      
      bs.setRanges(ranges);
      
      for (Entry<Key,Value> entry : bs) {
        found.put(entry.getKey().getRow(), entry.getValue());
      }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchScanner

mil.nga.giat.geowave.accumulo.metadata.AbstractAccumuloPersistence

mil.nga.giat.geowave.vector.query.DistributedRenderQuery

org.apache.accumulo.core.client.mock.MockConnectorTest

org.apache.accumulo.core.client.mock.MockTableOperationsTest

org.apache.accumulo.core.client.mock.TestBatchScanner821

org.apache.accumulo.core.iterators.user.IntersectingIteratorTest

org.apache.accumulo.core.iterators.user.TransformingIteratorTest

org.apache.accumulo.core.util.shell.commands.GrepCommand

org.apache.accumulo.core.util.shell.Shell$GrepCommand

org.apache.accumulo.examples.client.RandomBatchScanner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.