Examples of org.apache.accumulo.core.client.BatchWriter

org.apache.accumulo.core.client.BatchWriter
Send Mutations to a single Table in Accumulo. When the user uses a Connector to create a BatchWriter, they specify how much memory and how many threads it should use. As the user adds mutations to the batch writer, it buffers them. Once the buffered mutations have used half of the user specified buffer, the mutations are dumped into the background to be written by a thread pool. If the user specified memory completely fills up, then writes are held. When a user calls flush, it does not return until all buffered mutations are written.

      final String table1 = "testTableRenameDataValidation_table1", table2 = "testTableRenameDataValidation_table2";


      TableOperations tops = connector.tableOperations();
      tops.create(table1);


      BatchWriter bw1 = mtbw.getBatchWriter(table1);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");


      bw1.addMutation(m1);


      tops.rename(table1, table2);
      tops.create(table1);


      BatchWriter bw2 = mtbw.getBatchWriter(table1);
      
      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");


      bw1.addMutation(m2);
      bw2.addMutation(m2);


      mtbw.close();


      Map<Entry<String,String>,String> table1Expectations = new HashMap<Entry<String,String>,String>();
      table1Expectations.put(Maps.immutableEntry("bar", "col1"), "val1");

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.rename(table1, newTable1);
      tops.rename(table2, newTable2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      bw1.addMutation(m2);
      bw2.addMutation(m2);


      mtbw.close();


      Map<Entry<String,String>,String> expectations = new HashMap<Entry<String,String>,String>();

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.rename(table1, newTable1);


      // MTBW is still caching this name to the correct table, but we should invalidate its cache
      // after seeing the rename
      try {
        bw1 = mtbw.getBatchWriter(table1);
        Assert.fail("Should not be able to find this table");
      } catch (TableNotFoundException e) {
        // pass
      }


      tops.rename(table2, newTable2);
      
      try {
        bw2 = mtbw.getBatchWriter(table2);
        Assert.fail("Should not be able to find this table");
      } catch (TableNotFoundException e) {
        //pass
      }
      
      bw1 = mtbw.getBatchWriter(newTable1);
      bw2 = mtbw.getBatchWriter(newTable2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      bw1.addMutation(m2);
      bw2.addMutation(m2);


      mtbw.close();


      Map<Entry<String,String>,String> expectations = new HashMap<Entry<String,String>,String>();

View Full Code Here

        String key = WalkingSecurity.get(state).getLastKey() + "1";
        Mutation m = new Mutation(new Text(key));
        for (String s : WalkingSecurity.get(state).getAuthsArray()) {
          m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(Constants.UTF8)));
        }
        BatchWriter writer = null;
        try {
          try {
            writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1));
          } catch (TableNotFoundException tnfe) {
            if (tableExists)
              throw new AccumuloException("Table didn't exist when it should have: " + tableName);
            return;
          }
          boolean works = true;
          try {
            writer.addMutation(m);
            writer.close();
          } catch (MutationsRejectedException mre) {
            // Currently no method for detecting reason for mre. Waiting on ACCUMULO-670
            // For now, just wait a second and go again if they can write!
            if (!canWrite)
              return;
            
            if (ambiguousZone) {
              Thread.sleep(1000);
              try {
                writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxWriteThreads(1));
                writer.addMutation(m);
                writer.close();
                writer = null;
              } catch (MutationsRejectedException mre2) {
                throw new AccumuloException("Mutation exception!", mre2);
              }
            }
          }
          if (works)
            for (String s : WalkingSecurity.get(state).getAuthsArray())
              WalkingSecurity.get(state).increaseAuthMap(s, 1);
        } finally {
          if (writer != null) {
            writer.close();
            writer = null;
          }
        }
        break;
      case BULK_IMPORT:

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.rename(table1, newTable1);
      tops.rename(table2, newTable2);

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.delete(table1);
      tops.delete(table2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      try {
        bw1.addMutation(m2);
        bw2.addMutation(m2);
      } catch (MutationsRejectedException e) {
        // Pass - Mutations might flush immediately
        mutationsRejected = true;
      }

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.offline(table1);
      tops.offline(table2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      try {
        bw1.addMutation(m2);
        bw2.addMutation(m2);
      } catch (MutationsRejectedException e) {
        // Pass -- Mutations might flush immediately and fail because of offline table
        mutationsRejected = true;
      }

View Full Code Here


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.offline(table1);


      try {

View Full Code Here

    write("bt1", 1, 0, 1000000000, 1000);
    write("bt2", 2, 0, 1000000000, 1000);
    write("bt3", 3, 0, 1000000000, 1000);
    
    // test inserting an empty key
    BatchWriter bw = getConnector().createBatchWriter("bt4", new BatchWriterConfig());
    Mutation m = new Mutation(new Text(""));
    m.put(new Text(""), new Text(""), new Value("foo1".getBytes(Constants.UTF8)));
    bw.addMutation(m);
    bw.close();
    getConnector().tableOperations().flush("bt4", null, null, true);
    
    for (String table : new String[]{"bt1", "bt2", "bt3"}) {
      getConnector().tableOperations().setProperty(table, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "false");
      getConnector().tableOperations().setProperty(table, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "false");

View Full Code Here

    return t2 - t1;
  }
  
  private void write(String table, int depth, long start, long end, int step) throws Exception {
    
    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
    
    for (long i = start; i < end; i += step) {
      String key = String.format("k_%010d", i);
      
      Mutation m = null;
      
      switch (depth) {
        case 1:
          m = new Mutation(new Text(key));
          m.put(new Text("cf"), new Text("cq"), new Value(Long.toString(i).getBytes(Constants.UTF8)));
          break;
        case 2:
          m = new Mutation(new Text("row"));
          m.put(new Text(key), new Text("cq"), new Value(Long.toString(i).getBytes(Constants.UTF8)));
          break;
        case 3:
          m = new Mutation(new Text("row"));
          m.put(new Text("cf"), new Text(key), new Value(Long.toString(i).getBytes(Constants.UTF8)));
          break;
      }
      
      bw.addMutation(m);
    }
    
    bw.close();
    
    getConnector().tableOperations().flush(table, null, null, true);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

clojure_accumulo.iterators.ClojureCombinerTest

clojure_accumulo.iterators.ClojureFilterTest

clojure_accumulo.iterators.ClojureMapperTest

org.apache.accumulo.core.client.admin.FindMaxTest

org.apache.accumulo.core.client.ClientSideIteratorTest

org.apache.accumulo.core.client.impl.TabletServerBatchDeleter

org.apache.accumulo.core.client.mapreduce.AccumuloInputFormatTest

org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat$AccumuloRecordWriter

org.apache.accumulo.core.client.mock.MockBatchDeleter

org.apache.accumulo.core.client.mock.MockConnectorTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.