Package org.apache.accumulo.core.client

Examples of org.apache.accumulo.core.client.BatchWriter


    return Collections.singletonList(new TableSetup("abc"));
  }
 
  @Override
  public void run() throws Exception {
    BatchWriter bw = getConnector().createBatchWriter("abc", new BatchWriterConfig());
   
    for (int i = 0; i < 100000; i++) {
      Mutation m = new Mutation(new Text(String.format("%08d", i)));
      for (int j = 0; j < 3; j++)
        m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(Constants.UTF8)));
     
      bw.addMutation(m);
    }
   
    bw.close();
   
    Scanner scanner = getConnector().createScanner("abc", new Authorizations());
    scanner.setBatchSize(1000);
   
    Iterator<Entry<Key,Value>> iter = scanner.iterator();
View Full Code Here


    }
  }

  private void createEntries(Opts opts) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {

    BatchWriter batchWriter = opts.getConnector().createBatchWriter(opts.getTableName(), new BatchWriterConfig());

    Mutation m = new Mutation("row");
    m.put("cf", "cq", "value");

    // Trace the write operation. Note, unless you flush the BatchWriter, you will not capture
    // the write operation as it is occurs asynchronously. You can optionally create additional Spans
    // within a given Trace as seen below around the flush
    Trace.on("Client Write");

    batchWriter.addMutation(m);
    Span flushSpan = Trace.start("Client Flush");
    batchWriter.flush();
    flushSpan.stop();

    // Use Trace.offNoFlush() if you don't want the operation to block.
    Trace.off();

    batchWriter.close();
  }
View Full Code Here

 
  @Override
  public void run() throws Exception {
    getConnector().tableOperations().create("scftt");
   
    BatchWriter bw = getConnector().createBatchWriter("scftt", new BatchWriterConfig());
   
    // create file in the tablet that has mostly column family 0, with a few entries for column family 1

    bw.addMutation(nm(0, 1, 0));
    for (int i = 1; i < 99999; i++) {
      bw.addMutation(nm(i * 2, 0, i));
    }
    bw.addMutation(nm(99999 * 2, 1, 99999));
    bw.flush();
   
    getConnector().tableOperations().flush("scftt", null, null, true);
   
    // create a file that has column family 1 and 0 interleaved
    for (int i = 0; i < 100000; i++) {
      bw.addMutation(nm(i * 2 + 1, i % 2 == 0 ? 0 : 1, i));
    }
    bw.close();
   
    getConnector().tableOperations().flush("scftt", null, null, true);
   
    Scanner scanner = getConnector().createScanner("scftt", Constants.NO_AUTHS);
   
View Full Code Here

  }
 
  @Override
  public void run() throws Exception {
   
    BatchWriter bw = getConnector().createBatchWriter("foo", new BatchWriterConfig());
   
    Mutation m = new Mutation(new Text("r1"));
    m.put(new Text("acf"), new Text("foo"), new Value("1".getBytes(Constants.UTF8)));
   
    bw.addMutation(m);
   
    bw.close();
   
    getConnector().tableOperations().flush("foo", null, null, false);
    UtilWaitThread.sleep(1000);
   
    // minc should fail, so there should be no files
    checkRFiles("foo", 1, 1, 0, 0);
   
    // try to scan table
    Scanner scanner = getConnector().createScanner("foo", Constants.NO_AUTHS);
   
    int count = 0;
    for (@SuppressWarnings("unused")
    Entry<Key,Value> entry : scanner) {
      count++;
    }
   
    if (count != 1)
      throw new Exception("Did not see expected # entries " + count);
   
    // remove the bad iterator
    getConnector().tableOperations().removeProperty("foo", Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.badi");
   
    UtilWaitThread.sleep(5000);
   
    // minc should complete
    checkRFiles("foo", 1, 1, 1, 1);
   
    count = 0;
    for (@SuppressWarnings("unused")
    Entry<Key,Value> entry : scanner) {
      count++;
    }
   
    if (count != 1)
      throw new Exception("Did not see expected # entries " + count);
   
    // now try putting bad iterator back and deleting the table
    getConnector().tableOperations().setProperty("foo", Property.TABLE_ITERATOR_PREFIX.getKey() + "minc.badi", "30," + BadIterator.class.getName());
    bw = getConnector().createBatchWriter("foo", new BatchWriterConfig());
    m = new Mutation(new Text("r2"));
    m.put(new Text("acf"), new Text("foo"), new Value("1".getBytes(Constants.UTF8)));
    bw.addMutation(m);
    bw.close();
   
    // make sure property is given time to propagate
    UtilWaitThread.sleep(1000);
   
    getConnector().tableOperations().flush("foo", null, null, false);
View Full Code Here

    test2("ct2", false);
    test2("ct3", true);
  }
 
  private void test1() throws Exception {
    BatchWriter bw = getConnector().createBatchWriter("ct", new BatchWriterConfig());
   
    Mutation mut1 = new Mutation(new Text("r1"));
    mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(Constants.UTF8)));
   
    bw.addMutation(mut1);
   
    // should not throw any exceptions
    bw.close();
   
    bw = getConnector().createBatchWriter("ct", new BatchWriterConfig());
   
    // create a mutation with a non numeric value
    Mutation mut2 = new Mutation(new Text("r1"));
    mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(Constants.UTF8)));
   
    bw.addMutation(mut2);
   
    boolean sawMRE = false;
   
    try {
      bw.close();
      // should not get here
      throw new Exception("Test failed, constraint did not catch bad mutation");
    } catch (MutationsRejectedException mre) {
      sawMRE = true;
     
      // verify constraint violation summary
      List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
     
      if (cvsl.size() != 1) {
        throw new Exception("Unexpected constraints");
      }
     
      for (ConstraintViolationSummary cvs : cvsl) {
        if (!cvs.constrainClass.equals("org.apache.accumulo.examples.simple.constraints.NumericValueConstraint")) {
          throw new Exception("Unexpected constraint class " + cvs.constrainClass);
        }
       
        if (cvs.numberOfViolatingMutations != 1) {
          throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
        }
      }
    }
   
    if (!sawMRE) {
      throw new Exception("Did not see MutationsRejectedException");
    }
   
    // verify mutation did not go through
    Scanner scanner = getConnector().createScanner("ct", Constants.NO_AUTHS);
    scanner.setRange(new Range(new Text("r1")));
   
    Iterator<Entry<Key,Value>> iter = scanner.iterator();
    Entry<Key,Value> entry = iter.next();
   
    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(Constants.UTF8)))) {
      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    if (iter.hasNext()) {
      entry = iter.next();
      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    // remove the numeric value constraint
    getConnector().tableOperations().removeConstraint("ct", 1);
    UtilWaitThread.sleep(1000);
   
    // now should be able to add a non numeric value
    bw = getConnector().createBatchWriter("ct", new BatchWriterConfig());
    bw.addMutation(mut2);
    bw.close();
   
    // verify mutation went through
    iter = scanner.iterator();
    entry = iter.next();
   
    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    if (iter.hasNext()) {
      entry = iter.next();
      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    // add a constraint that references a non-existant class
    getConnector().tableOperations().setProperty("ct", Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
    UtilWaitThread.sleep(1000);
   
    // add a mutation
    bw = getConnector().createBatchWriter("ct", new BatchWriterConfig());
   
    Mutation mut3 = new Mutation(new Text("r1"));
    mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(Constants.UTF8)));
   
    bw.addMutation(mut3);
   
    sawMRE = false;
   
    try {
      bw.close();
      // should not get here
      throw new Exception("Test failed, mutation went through when table had bad constraints");
    } catch (MutationsRejectedException mre) {
      sawMRE = true;
    }
   
    if (!sawMRE) {
      throw new Exception("Did not see MutationsRejectedException");
    }
   
    // verify the mutation did not go through
    iter = scanner.iterator();
    entry = iter.next();
   
    if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1"))
        || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(Constants.UTF8)))) {
      throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    if (iter.hasNext()) {
      entry = iter.next();
      throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
    }
   
    // remove the bad constraint
    getConnector().tableOperations().removeProperty("ct", Property.TABLE_CONSTRAINT_PREFIX + "1");
    UtilWaitThread.sleep(1000);
   
    // try the mutation again
    bw = getConnector().createBatchWriter("ct", new BatchWriterConfig());
    bw.addMutation(mut3);
    bw.close();
   
    // verify it went through
    iter = scanner.iterator();
    entry = iter.next();
   
View Full Code Here

  private void test2(String table, boolean doFlush) throws Exception {
    // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
    // should go through
    int numericErrors = 2;
   
    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
    bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
    bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
    if (doFlush) {
      try {
        bw.flush();
        throw new Exception("Didn't find a bad mutation");
      } catch (MutationsRejectedException mre) {
        // ignored
        try {
          bw.close();
        } catch (MutationsRejectedException ex) {
          // ignored
        }
        bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
        numericErrors = 1;
      }
    }
    bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
    bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
    bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
   
    boolean sawMRE = false;
   
    try {
      bw.close();
      // should not get here
      throw new Exception("Test failed, constraint did not catch bad mutation");
    } catch (MutationsRejectedException mre) {
      System.out.println(mre);
     
View Full Code Here

   
    c.tableOperations().setProperty(table1, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1M");
    c.tableOperations().setProperty(table1, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX.getKey(), "2M");
    c.tableOperations().setProperty(table1, Property.TABLE_FILE_MAX.getKey(), "23");
   
    BatchWriter bw = c.createBatchWriter(table1, new BatchWriterConfig());
   
    Mutation m1 = new Mutation("001");
    m1.put("data", "x", "9");
    m1.put("data", "y", "7");
   
    Mutation m2 = new Mutation("008");
    m2.put("data", "x", "3");
    m2.put("data", "y", "4");
   
    bw.addMutation(m1);
    bw.addMutation(m2);
   
    bw.flush();
   
    Map<String,String> props = new HashMap<String,String>();
    props.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "500K");
   
    Set<String> exclude = new HashSet<String>();
    exclude.add(Property.TABLE_FILE_MAX.getKey());
   
    c.tableOperations().clone(table1, table2, true, props, exclude);
   
    Mutation m3 = new Mutation("009");
    m3.put("data", "x", "1");
    m3.put("data", "y", "2");
    bw.addMutation(m3);
    bw.close();

    Scanner scanner = c.createScanner(table2, Constants.NO_AUTHS);
   
    HashMap<String,String> expected = new HashMap<String,String>();
    expected.put("001:x", "9");
View Full Code Here

   
  }
 
  private void runLatencyTest() throws Exception {
    // should automatically flush after 3 seconds
    BatchWriter bw = getConnector().createBatchWriter("bwlt", new BatchWriterConfig().setMaxLatency(2000, TimeUnit.MILLISECONDS));
    Scanner scanner = getConnector().createScanner("bwlt", Constants.NO_AUTHS);
   
    Mutation m = new Mutation(new Text(String.format("r_%10d", 1)));
    m.put(new Text("cf"), new Text("cq"), new Value("1".getBytes(Constants.UTF8)));
    bw.addMutation(m);
   
    UtilWaitThread.sleep(1000);
   
    int count = 0;
    for (@SuppressWarnings("unused")
    Entry<Key,Value> entry : scanner) {
      count++;
    }
   
    if (count != 0) {
      throw new Exception("Flushed too soon");
    }
   
    UtilWaitThread.sleep(4000);
   
    for (@SuppressWarnings("unused")
    Entry<Key,Value> entry : scanner) {
      count++;
    }
   
    if (count != 1) {
      throw new Exception("Did not flush");
    }
   
    bw.close();
  }
View Full Code Here

   
    bw.close();
  }
 
  private void runFlushTest() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException, Exception {
    BatchWriter bw = getConnector().createBatchWriter("bwft", new BatchWriterConfig());
    Scanner scanner = getConnector().createScanner("bwft", Constants.NO_AUTHS);
    Random r = new Random();
   
    for (int i = 0; i < 4; i++) {
      for (int j = 0; j < NUM_TO_FLUSH; j++) {
        int row = i * NUM_TO_FLUSH + j;
       
        Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
        m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
        bw.addMutation(m);
      }
     
      bw.flush();
     
      // do a few random lookups into the data just flushed
     
      for (int k = 0; k < 10; k++) {
        int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
       
        scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
       
        Iterator<Entry<Key,Value>> iter = scanner.iterator();
       
        if (!iter.hasNext())
          throw new Exception(" row " + rowToLookup + " not found after flush");
       
        Entry<Key,Value> entry = iter.next();
       
        if (iter.hasNext())
          throw new Exception("Scanner returned too much");
       
        verifyEntry(rowToLookup, entry);
      }
     
      // scan all data just flushed
      scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
      Iterator<Entry<Key,Value>> iter = scanner.iterator();
     
      for (int j = 0; j < NUM_TO_FLUSH; j++) {
        int row = i * NUM_TO_FLUSH + j;
       
        if (!iter.hasNext())
          throw new Exception("Scan stopped permaturely at " + row);
       
        Entry<Key,Value> entry = iter.next();
       
        verifyEntry(row, entry);
      }
     
      if (iter.hasNext())
        throw new Exception("Scanner returned too much");
     
    }
   
    bw.close();
   
    // test adding a mutation to a closed batch writer
    boolean caught = false;
    try {
      bw.addMutation(new Mutation(new Text("foobar")));
    } catch (IllegalStateException ise) {
      caught = true;
    }
   
    if (!caught) {
View Full Code Here

    final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";

    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
    Connector c = mockInstance.getConnector("root", new PasswordToken(""));
    c.tableOperations().create(TEST_TABLE_1);
    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();

    Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME, AccumuloInputFormat.class.getCanonicalName()}));
    assertNull(e1);
    assertNull(e2);
  }
View Full Code Here

TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.