Package org.apache.accumulo.core.client

Examples of org.apache.accumulo.core.client.Connector


  }
 
  @Test
  public void test() throws Exception {
    Instance instance = new MockInstance();
    Connector connector = instance.getConnector("root", new PasswordToken(""));
    BatchWriter bw = connector.createBatchWriter("!METADATA", new BatchWriterConfig());
   
    // Create a fake METADATA table with these splits
    String splits[] = {"a", "e", "j", "o", "t", "z"};
    // create metadata for a table "t" with the splits above
    Text tableId = new Text("t");
    Text pr = null;
    for (String s : splits) {
      Text split = new Text(s);
      Mutation prevRow = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, split, pr));
      prevRow.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
      Constants.METADATA_CHOPPED_COLUMN.put(prevRow, new Value("junk".getBytes()));
      bw.addMutation(prevRow);
      pr = split;
    }
    // Add the default tablet
    Mutation defaultTablet = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, null, pr));
    defaultTablet.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
    bw.addMutation(defaultTablet);
    bw.close();
   
    // Read out the TabletLocationStates
    MockCurrentState state = new MockCurrentState(new MergeInfo(new KeyExtent(tableId, new Text("p"), new Text("e")), MergeInfo.Operation.MERGE));
    TCredentials auths = CredentialHelper.create("root", new PasswordToken(new byte[0]), "instance");
   
    // Verify the tablet state: hosted, and count
    MetaDataStateStore metaDataStateStore = new MetaDataStateStore(instance, auths, state);
    int count = 0;
    for (TabletLocationState tss : metaDataStateStore) {
      Assert.assertEquals(TabletState.HOSTED, tss.getState(state.onlineTabletServers()));
      count++;
    }
    Assert.assertEquals(splits.length + 1, count);
   
    // Create the hole
    // Split the tablet at one end of the range
    Mutation m = new KeyExtent(tableId, new Text("t"), new Text("p")).getPrevRowUpdateMutation();
    Constants.METADATA_SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
    Constants.METADATA_OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(new Text("o")));
    update(connector, m);
   
    // do the state check
    MergeStats stats = scan(state, metaDataStateStore);
    MergeState newState = stats.nextMergeState(connector, state);
    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
   
    // unassign the tablets
    BatchDeleter deleter = connector.createBatchDeleter("!METADATA", Constants.NO_AUTHS, 1000, new BatchWriterConfig());
    deleter.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
    deleter.setRanges(Collections.singletonList(new Range()));
    deleter.delete();
   
    // now we should be ready to merge but, we have an inconsistent !METADATA table
View Full Code Here


    System.arraycopy(b1, 0, ball, 0, b1.length);
    ball[b1.length] = (byte) 0;
    System.arraycopy(b2, 0, ball, b1.length + 1, b2.length);

    Instance instance = new MockInstance();
    Connector conn = instance.getConnector("root", new PasswordToken(new byte[0]));

    conn.tableOperations().create(table);
    BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
    Mutation m = new Mutation(ball);
    m.put(new byte[0], new byte[0], new byte[0]);
    bw.addMutation(m);
    bw.close();

    IteratorSetting is = new IteratorSetting(5, RegExFilter.class);
    RegExFilter.setRegexs(is, s2, null, null, null, true, true);

    Scanner scanner = conn.createScanner(table, new Authorizations());
    scanner.addScanIterator(is);

    assertTrue("Client side iterator couldn't find a match when it should have", scanner.iterator().hasNext());

    conn.tableOperations().attachIterator(table, is);
    assertTrue("server side iterator couldn't find a match when it should have", conn.createScanner(table, new Authorizations()).iterator().hasNext());
  }
View Full Code Here

    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(TestBinaryRows.class.getName(), args, scanOpts, bwOpts);
   
    try {
      Connector connector = opts.getConnector();
     
      final Text CF = new Text("cf"), CQ = new Text("cq");
      final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
     
      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
        BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
        boolean delete = opts.mode.equals("delete");
       
        for (long i = 0; i < opts.num; i++) {
          byte[] row = encodeLong(i + opts.start);
          String value = "" + (i + opts.start);
         
          Mutation m = new Mutation(new Text(row));
          if (delete) {
            m.putDelete(CF, CQ);
          } else {
            m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
          }
          bw.addMutation(m);
        }
       
        bw.close();
      } else if (opts.mode.equals("verifyDeleted")) {
        Scanner s = connector.createScanner(opts.tableName, opts.auths);
        s.setBatchSize(scanOpts.scanBatchSize);
        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
        s.setBatchSize(50000);
        s.setRange(new Range(startKey, stopKey));
       
        for (Entry<Key,Value> entry : s) {
          System.err.println("ERROR : saw entries in range that should be deleted ( first value : " + entry.getValue().toString() + ")");
          System.err.println("exiting...");
          System.exit(1);
        }
       
      } else if (opts.mode.equals("verify")) {
        long t1 = System.currentTimeMillis();
       
        Scanner s = connector.createScanner(opts.tableName, opts.auths);
        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);
        s.setBatchSize(scanOpts.scanBatchSize);
        s.setRange(new Range(startKey, stopKey));
       
        long i = opts.start;
       
        for (Entry<Key,Value> e : s) {
          Key k = e.getKey();
          Value v = e.getValue();
         
          // System.out.println("v = "+v);
         
          checkKeyValue(i, k, v);
         
          i++;
        }
       
        if (i != opts.start + opts.num) {
          System.err.println("ERROR : did not see expected number of rows, saw " + (i - opts.start) + " expected " + opts.num);
          System.err.println("exiting... ARGHHHHHH");
          System.exit(1);
         
        }
       
        long t2 = System.currentTimeMillis();
       
        System.out.printf("time : %9.2f secs%n", ((t2 - t1) / 1000.0));
        System.out.printf("rate : %9.2f entries/sec%n", opts.num / ((t2 - t1) / 1000.0));
       
      } else if (opts.mode.equals("randomLookups")) {
        int numLookups = 1000;
       
        Random r = new Random();
       
        long t1 = System.currentTimeMillis();
       
        for (int i = 0; i < numLookups; i++) {
          long row = ((r.nextLong() & 0x7fffffffffffffffl) % opts.num) + opts.start;
         
          Scanner s = connector.createScanner(opts.tableName, opts.auths);
          s.setBatchSize(scanOpts.scanBatchSize);
          Key startKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
          Key stopKey = new Key(encodeLong(row), CF_BYTES, CQ_BYTES, new byte[0], 0);
          s.setRange(new Range(startKey, stopKey));
         
          Iterator<Entry<Key,Value>> si = s.iterator();
         
          if (si.hasNext()) {
            Entry<Key,Value> e = si.next();
            Key k = e.getKey();
            Value v = e.getValue();
           
            checkKeyValue(row, k, v);
           
            if (si.hasNext()) {
              System.err.println("ERROR : lookup on " + row + " returned more than one result ");
              System.err.println("exiting...");
              System.exit(1);
            }
           
          } else {
            System.err.println("ERROR : lookup on " + row + " failed ");
            System.err.println("exiting...");
            System.exit(1);
          }
        }
       
        long t2 = System.currentTimeMillis();
       
        System.out.printf("time    : %9.2f secs%n", ((t2 - t1) / 1000.0));
        System.out.printf("lookups : %9d keys%n", numLookups);
        System.out.printf("rate    : %9.2f lookups/sec%n", numLookups / ((t2 - t1) / 1000.0));
       
      } else if (opts.mode.equals("split")) {
        TreeSet<Text> splits = new TreeSet<Text>();
        int shift = (int) opts.start;
        int count = (int) opts.num;
       
        for (long i = 0; i < count; i++) {
          long splitPoint = i << shift;
         
          splits.add(new Text(encodeLong(splitPoint)));
          System.out.printf("added split point 0x%016x  %,12d%n", splitPoint, splitPoint);
        }
       
        connector.tableOperations().create(opts.tableName);
        connector.tableOperations().addSplits(opts.tableName, splits);
       
      } else {
        System.err.println("ERROR : " + opts.mode + " is not a valid operation.");
        System.exit(1);
      }
View Full Code Here

    Opts opts = new Opts();
    ScannerOpts scanOpts = new ScannerOpts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(TestMultiTableIngest.class.getName(), args, scanOpts, bwOpts);
    // create the test table within accumulo
    Connector connector;
    try {
      connector = opts.getConnector();
    } catch (AccumuloException e) {
      throw new RuntimeException(e);
    } catch (AccumuloSecurityException e) {
      throw new RuntimeException(e);
    }
    for (int i = 0; i < opts.tables; i++) {
      tableNames.add(String.format("test_%04d", i));
    }
   
    if (!opts.readonly) {
      for (String table : tableNames)
        connector.tableOperations().create(table);
     
      MultiTableBatchWriter b;
      try {
        b = connector.createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
     
      // populate
View Full Code Here

      at = token;
    }
   
    String table = conf.get(Property.TRACE_TABLE);
    try {
      Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
      if (!conn.tableOperations().exists(table)) {
        return new NullScanner();
      }
      Scanner scanner = conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
      return scanner;
    } catch (AccumuloSecurityException ex) {
      sb.append("<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
      return null;
    } catch (TableNotFoundException ex) {
View Full Code Here

    BatchWriterOpts bwOpts = new BatchWriterOpts();
    ScannerOpts scanOpts = new ScannerOpts();
    opts.parseArgs(program, args, bwOpts, scanOpts);
   
    // create the test table within accumulo
    Connector connector = opts.getConnector();
   
    if (!opts.readOnly) {
      TreeSet<Text> keys = new TreeSet<Text>();
      for (int i = 0; i < opts.count / 100; i++) {
        keys.add(new Text(String.format("%05d", i * 100)));
      }
     
      // presplit
      connector.tableOperations().create(opts.getTableName());
      connector.tableOperations().addSplits(opts.getTableName(), keys);
      BatchWriter b = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
     
      // populate
      for (int i = 0; i < opts.count; i++) {
        Mutation m = new Mutation(new Text(String.format("%05d", i)));
        m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
View Full Code Here

  public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(ContinuousWalk.class.getName(), args);
   
    Connector conn = opts.getConnector();
   
    Random r = new Random();
   
    ArrayList<Value> values = new ArrayList<Value>();
   
View Full Code Here

    job.setInputFormatClass(AccumuloInputFormat.class);
    opts.setAccumuloConfigs(job);

    Set<Range> ranges = null;
    String clone = opts.getTableName();
    Connector conn = null;

    if (opts.scanOffline) {
      Random random = new Random();
      clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
      conn = opts.getConnector();
      conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
      ranges = conn.tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
      conn.tableOperations().offline(clone);
      AccumuloInputFormat.setInputTableName(job, clone);
      AccumuloInputFormat.setOfflineTableScan(job, true);
    } else {
      ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
    }

    AccumuloInputFormat.setRanges(job, ranges);
    AccumuloInputFormat.setAutoAdjustRanges(job, false);

    job.setMapperClass(CMapper.class);
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(VLongWritable.class);

    job.setReducerClass(CReducer.class);
    job.setNumReduceTasks(opts.reducers);

    job.setOutputFormatClass(TextOutputFormat.class);

    job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);

    TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));

    job.waitForCompletion(true);

    if (opts.scanOffline) {
      conn.tableOperations().delete(clone);
    }
    opts.stopTracing();
    return job.isSuccessful() ? 0 : 1;
  }
View Full Code Here

        throw new IllegalArgumentException("Can not import files to root tablet");
      }

      synchronized (bulkFileImportLock) {
        TCredentials auths = SecurityConstants.getSystemCredentials();
        Connector conn;
        try {
          conn = HdfsZooInstance.getInstance().getConnector(auths.getPrincipal(), CredentialHelper.extractToken(auths));
        } catch (Exception ex) {
          throw new IOException(ex);
        }
View Full Code Here

      String ref = tokens[1];
     
      undefs.add(new UndefinedNode(undef, ref));
    }
   
    Connector conn = opts.getConnector();
    BatchScanner bscanner = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads);
    bscanner.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
    List<Range> refs = new ArrayList<Range>();
   
    for (UndefinedNode undefinedNode : undefs)
      refs.add(new Range(new Text(undefinedNode.ref)));
   
    bscanner.setRanges(refs);
   
    HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>();
   
    for (Entry<Key,Value> entry : bscanner) {
      String ref = entry.getKey().getRow().toString();
      List<String> vals = refInfo.get(ref);
      if (vals == null) {
        vals = new ArrayList<String>();
        refInfo.put(ref, vals);
      }
     
      vals.add(entry.getValue().toString());
    }
   
    bscanner.close();
   
    IngestInfo ingestInfo = new IngestInfo(opts.logDir);
    TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(conn.getInstance(), opts.getTableName()), opts.logDir);
   
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
   
    for (UndefinedNode undefinedNode : undefs) {
     
View Full Code Here

TOP

Related Classes of org.apache.accumulo.core.client.Connector

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.