Examples of org.apache.accumulo.core.client.MultiTableBatchWriter

org.apache.accumulo.core.client.MultiTableBatchWriter
This class enables efficient batch writing to multiple tables. When creating a batch writer for each table, each has its own memory and network resources. Using this class these resources may be shared among multiple tables.

    Connector connector = instance.getConnector("root", password);


    BatchWriterConfig config = new BatchWriterConfig();


    TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID());
    MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS);
    boolean mutationsRejected = false;
    
    try {
      final String table1 = "testTableDelete_table1", table2 = "testTableDelete_table2";


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.delete(table1);
      tops.delete(table2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      try {
        bw1.addMutation(m2);
        bw2.addMutation(m2);
      } catch (MutationsRejectedException e) {
        // Pass - Mutations might flush immediately
        mutationsRejected = true;
      }


    } finally {
      if (null != mtbw) {
        try {
          // Mutations might have flushed before the table offline occurred
          mtbw.close();
        } catch (MutationsRejectedException e) {
          // Pass
          mutationsRejected = true;
        }
      }

View Full Code Here

    Connector connector = instance.getConnector("root", password);


    BatchWriterConfig config = new BatchWriterConfig();


    TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID());
    MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS);
    boolean mutationsRejected = false;


    try {
      final String table1 = "testOfflineTable_table1", table2 = "testOfflineTable_table2";


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.offline(table1);
      tops.offline(table2);


      Mutation m2 = new Mutation("bar");
      m2.put("col1", "", "val1");
      m2.put("col2", "", "val2");


      try {
        bw1.addMutation(m2);
        bw2.addMutation(m2);
      } catch (MutationsRejectedException e) {
        // Pass -- Mutations might flush immediately and fail because of offline table
        mutationsRejected = true;
      }
    } finally {
      if (null != mtbw) {
        try {
          // Mutations might have flushed before the table offline occurred
          mtbw.close();
        } catch (MutationsRejectedException e) {
          // Pass
          mutationsRejected = true;
        }
      }

View Full Code Here

    Connector connector = instance.getConnector("root", password);


    BatchWriterConfig config = new BatchWriterConfig();
    
    TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID());
    MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 60, TimeUnit.SECONDS);
    boolean mutationsRejected = false;


    try {
      final String table1 = "testOfflineTableWithCache_table1", table2 = "testOfflineTableWithCache_table2";


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      tops.offline(table1);


      try {
        bw1 = mtbw.getBatchWriter(table1);
      } catch (TableOfflineException e) {
        // pass
        mutationsRejected = true;
      }


      tops.offline(table2);


      try {
        bw2 = mtbw.getBatchWriter(table2);
      } catch (TableOfflineException e) {
        // pass
        mutationsRejected = true;
      }
    } finally {
      if (null != mtbw) {
        try {
          // Mutations might have flushed before the table offline occurred
          mtbw.close();
        } catch (MutationsRejectedException e) {
          // Pass
          mutationsRejected = true;
        }
      }

View Full Code Here

    Connector connector = instance.getConnector("root", password);


    BatchWriterConfig config = new BatchWriterConfig();


    TCredentials creds = CredentialHelper.create("root", password, instance.getInstanceID());
    MultiTableBatchWriter mtbw = new MultiTableBatchWriterImpl(instance, creds, config, 0, TimeUnit.SECONDS);
    boolean mutationsRejected = false;


    try {
      final String table1 = "testOfflineTableWithoutCache_table1", table2 = "testOfflineTableWithoutCache_table2";


      TableOperations tops = connector.tableOperations();
      tops.create(table1);
      tops.create(table2);


      BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);


      Mutation m1 = new Mutation("foo");
      m1.put("col1", "", "val1");
      m1.put("col2", "", "val2");


      bw1.addMutation(m1);
      bw2.addMutation(m1);


      // Mutations might or might not flush before tables goes offline
      tops.offline(table1);
      tops.offline(table2);


      try {
        bw1 = mtbw.getBatchWriter(table1);
        Assert.fail(table1 + " should be offline");
      } catch (TableOfflineException e) {
        // pass
        mutationsRejected = true;
      }


      try {
        bw2 = mtbw.getBatchWriter(table2);
        Assert.fail(table1 + " should be offline");
      } catch (TableOfflineException e) {
        // pass
        mutationsRejected = true;
      }
    } finally {
      if (null != mtbw) {
        try {
          // Mutations might have flushed before the table offline occurred
          mtbw.close();
        } catch (MutationsRejectedException e) {
          // Pass
          mutationsRejected = true;
        }
      }

View Full Code Here

    KeyExtent last = new KeyExtent();
    String directory = null;
    Set<String> knownFiles = new HashSet<String>();
    
    int count = 0;
    final MultiTableBatchWriter writer = opts.getConnector().createMultiTableBatchWriter(bwOpts.getBatchWriterConfig());
    
    // collect the list of known files and the directory for each extent
    for (Entry<Key,Value> entry : scanner) {
      Key key = entry.getKey();
      KeyExtent ke = new KeyExtent(key.getRow(), (Text) null);
      // when the key extent changes
      if (!ke.equals(last)) {
        if (directory != null) {
          // add any files in the directory unknown to the key extent
          count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
        }
        directory = null;
        knownFiles.clear();
        last = ke;
      }
      if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
        directory = entry.getValue().toString();
        log.debug("Found directory " + directory + " for row " + key.getRow().toString());
      } else if (key.compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY) == 0) {
        String filename = key.getColumnQualifier().toString();
        knownFiles.add(filename);
        log.debug("METADATA file found: " + filename);
      }
    }
    if (directory != null) {
      // catch the last key extent
      count += addUnknownFiles(fs, directory, knownFiles, last, writer, opts.update);
    }
    log.info("There were " + count + " files that are unknown to the metadata table");
    writer.close();
  }

View Full Code Here

  
  @Override
  public void tearDown(State state) throws Exception {
    // We have resources we need to clean up
    if (state.isMultiTableBatchWriterInitialized()) {
      MultiTableBatchWriter mtbw = state.getMultiTableBatchWriter();
      try {
        mtbw.close();
      } catch (MutationsRejectedException e) {
        log.error("Ignoring mutations that weren't flushed", e);
      }


      // Reset the MTBW on the state to null

View Full Code Here

  
  @Override
  public void tearDown(State state) throws Exception {
    // We have resources we need to clean up
    if (state.isMultiTableBatchWriterInitialized()) {
      MultiTableBatchWriter mtbw = state.getMultiTableBatchWriter();
      try {
        mtbw.close();
      } catch (MutationsRejectedException e) {
        log.error("Ignoring mutations that weren't flushed", e);
      }


      // Reset the MTBW on the state to null

View Full Code Here

  
  @Override
  public void tearDown(State state) throws Exception {
    // We have resources we need to clean up
    if (state.isMultiTableBatchWriterInitialized()) {
      MultiTableBatchWriter mtbw = state.getMultiTableBatchWriter();
      try {
        mtbw.close();
      } catch (MutationsRejectedException e) {
        log.error("Ignoring mutations that weren't flushed", e);
      }


      // Reset the MTBW on the state to null

View Full Code Here

  @Test
  public void testMockMultiTableBatchWriter() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("a");
    c.tableOperations().create("b");
    MultiTableBatchWriter bw = c.createMultiTableBatchWriter(10000L, 1000L, 4);
    Mutation m1 = new Mutation("r1");
    m1.put("cf1", "cq1", 1, "v1");
    BatchWriter b = bw.getBatchWriter("a");
    b.addMutation(m1);
    b.flush();
    b = bw.getBatchWriter("b");
    b.addMutation(m1);
    b.flush();
    
    Scanner scanner = c.createScanner("a", Constants.NO_AUTHS);
    int count = 0;

View Full Code Here

  
  @Override
  public void tearDown(State state) throws Exception {
    // We have resources we need to clean up
    if (state.isMultiTableBatchWriterInitialized()) {
      MultiTableBatchWriter mtbw = state.getMultiTableBatchWriter();
      try {
        mtbw.close();
      } catch (MutationsRejectedException e) {
        log.error("Ignoring mutations that weren't flushed", e);
      }


      // Reset the MTBW on the state to null

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.accumulo.core.client.MultiTableBatchWriter

org.apache.accumulo.core.client.mock.MockConnectorTest

org.apache.accumulo.examples.helloworld.InsertWithBatchWriter

org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter

org.apache.accumulo.server.test.randomwalk.image.ImageFixture

org.apache.accumulo.server.test.randomwalk.image.Write

org.apache.accumulo.server.test.randomwalk.multitable.MultiTableFixture

org.apache.accumulo.server.test.randomwalk.sequential.SequentialFixture

org.apache.accumulo.server.test.randomwalk.shard.ShardFixture

org.apache.accumulo.server.test.TestMultiTableIngest

org.apache.accumulo.server.util.AddFilesWithMissingEntries

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.