Examples of org.apache.accumulo.core.client.BatchWriter

org.apache.accumulo.core.client.BatchWriter
Send Mutations to a single Table in Accumulo. When the user uses a Connector to create a BatchWriter, they specify how much memory and how many threads it should use. As the user adds mutations to the batch writer, it buffers them. Once the buffered mutations have used half of the user specified buffer, the mutations are dumped into the background to be written by a thread pool. If the user specified memory completely fills up, then writes are held. When a user calls flush, it does not return until all buffered mutations are written.

    PasswordToken password = new PasswordToken("");


    MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
    Connector c = mockInstance.getConnector(user, password);
    c.tableOperations().create("testtable");
    BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();


    Assert.assertEquals(
        0,
        MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
            EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));

View Full Code Here

    PasswordToken password = new PasswordToken("");


    MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
    Connector c = mockInstance.getConnector(user, password);
    c.tableOperations().create("testtable");
    BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();


    // We should fail before we even get into the Mapper because we can't make the RecordReader
    Assert.assertEquals(
        1,
        MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",

View Full Code Here

    metadata.setBatchSize(scanOpts.scanBatchSize);
    metadata.setRange(Constants.METADATA_KEYSPACE);
    metadata.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
    int count = 0;
    int missing = 0;
    BatchWriter writer = null; 
    if (opts.fix)
      writer = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, bwOpts.getBatchWriterConfig());
    for (Entry<Key,Value> entry : metadata) {
      count++;
      Key key = entry.getKey();
      String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()), Constants.UTF8);
      String file = key.getColumnQualifier().toString();
      if (!file.startsWith("/"))
        file = "/" + file;
      Path map = new Path(ServerConstants.getTablesDir() + "/" + table + file);
      if (!fs.exists(map)) {
        missing++;
        log.info("File " + map + " is missing");
        Mutation m = new Mutation(key.getRow());
        m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
        if (writer != null) {
          writer.addMutation(m);
          log.info("entry removed from metadata table: " + m);
        }
      }
    }
    if (writer != null && missing > 0)
      writer.close();
    log.info(String.format("%d files of %d missing", missing, count));
  }

View Full Code Here

  
  @Test
  public void testSunnyDay() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
      int r = random.nextInt();
      Mutation m = new Mutation(asText(r));
      m.put(asText(random.nextInt()), asText(random.nextInt()), new Value(Integer.toHexString(r).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    BatchScanner s = c.createBatchScanner("test", Constants.NO_AUTHS, 2);
    s.setRanges(Collections.singletonList(new Range()));
    Key key = null;
    int count = 0;
    for (Entry<Key,Value> entry : s) {

View Full Code Here

    server.serve();
  }
  
  private void flush() {
    try {
      final BatchWriter writer = this.writer.get();
      if (null != writer) {
        writer.flush();
      }
    } catch (MutationsRejectedException exception) {
      log.warn("Problem flushing traces, resetting writer. Set log level to DEBUG to see stacktrace. cause: " + exception);
      log.debug("flushing traces failed due to exception", exception);
      resetWriter();

View Full Code Here

  
  @Test
  public void testBadMutations() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", new BatchWriterConfig().setMaxMemory(10000L).setMaxLatency(1000L, TimeUnit.MILLISECONDS).setMaxWriteThreads(4));


    try {
      bw.addMutation(null);
      Assert.fail("addMutation should throw IAE for null mutation");
    } catch (IllegalArgumentException iae) {}
    try {
      bw.addMutations(null);
      Assert.fail("addMutations should throw IAE for null iterable");
    } catch (IllegalArgumentException iae) {}


    bw.addMutations(Collections.<Mutation>emptyList());


    Mutation bad = new Mutation("bad");
    try {
      bw.addMutation(bad);
      Assert.fail("addMutation should throw IAE for empty mutation");
    } catch (IllegalArgumentException iae) {}




    Mutation good = new Mutation("good");
    good.put(asText(random.nextInt()), asText(random.nextInt()), new Value("good".getBytes()));
    List<Mutation> mutations = new ArrayList<Mutation>();
    mutations.add(good);
    mutations.add(bad);
    try {
      bw.addMutations(mutations);
      Assert.fail("addMutations should throw IAE if it contains empty mutation");
    } catch (IllegalArgumentException iae) {}


    bw.close();
  }

View Full Code Here

      resetWriter();
    }
  }
  
  private void resetWriter() {
    BatchWriter writer = null;
    try {
      writer = connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(5, TimeUnit.SECONDS));
    } catch (Exception ex) {
      log.warn("Unable to create a batch writer, will retry. Set log level to DEBUG to see stacktrace. cause: " + ex);
      log.debug("batch writer creation failed with exception.", ex);
    } finally {
      /* Trade in the new writer (even if null) for the one we need to close. */
      writer = this.writer.getAndSet(writer);
      try {
        if (null != writer) {
          writer.close();
        }
      } catch (Exception ex) {
        log.warn("Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex);
        log.debug("batch writer close failed with exception", ex);
      }

View Full Code Here

    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("day")));
    SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    c.tableOperations().attachIterator(table, is);
    String keys[][] = { {"foo", "day", "20080101"}, {"foo", "day", "20080101"}, {"foo", "day", "20080103"}, {"bar", "day", "20080101"},
        {"bar", "day", "20080101"},};
    BatchWriter bw = c.createBatchWriter("perDayCounts", new BatchWriterConfig());
    for (String elt[] : keys) {
      Mutation m = new Mutation(new Text(elt[0]));
      m.put(new Text(elt[1]), new Text(elt[2]), new Value("1".getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    
    Scanner s = c.createScanner("perDayCounts", Constants.NO_AUTHS);
    Iterator<Entry<Key,Value>> iterator = s.iterator();
    assertTrue(iterator.hasNext());
    checkEntry(iterator.next(), "bar", "day", "20080101", "2");

View Full Code Here

  
  @Test
  public void testDelete() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", new BatchWriterConfig());
    
    Mutation m1 = new Mutation("r1");
    
    m1.put("cf1", "cq1", 1, "v1");
    
    bw.addMutation(m1);
    bw.flush();
    
    Mutation m2 = new Mutation("r1");
    
    m2.putDelete("cf1", "cq1", 2);
    
    bw.addMutation(m2);
    bw.flush();
    
    Scanner scanner = c.createScanner("test", Constants.NO_AUTHS);
    
    int count = 0;
    for (@SuppressWarnings("unused")

View Full Code Here

    
    // ensure there is no data from previous test
    Assert.assertEquals(0, count);
    
    Connector conn = instance.getConnector(credential.getPrincipal(), CredentialHelper.extractToken(credential));
    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
    for (String line : metadata) {
      String[] parts = line.split(" ");
      String[] columnParts = parts[1].split(":");
      Mutation m = new Mutation(parts[0]);
      m.put(new Text(columnParts[0]), new Text(columnParts[1]), new Value(parts[2].getBytes()));
      bw.addMutation(m);
    }
    
    for (String line : deletes) {
      Mutation m = new Mutation(line);
      m.put("", "", "");
      bw.addMutation(m);
    }
    bw.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

clojure_accumulo.iterators.ClojureCombinerTest

clojure_accumulo.iterators.ClojureFilterTest

clojure_accumulo.iterators.ClojureMapperTest

org.apache.accumulo.core.client.admin.FindMaxTest

org.apache.accumulo.core.client.ClientSideIteratorTest

org.apache.accumulo.core.client.impl.TabletServerBatchDeleter

org.apache.accumulo.core.client.mapreduce.AccumuloInputFormatTest

org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat$AccumuloRecordWriter

org.apache.accumulo.core.client.mock.MockBatchDeleter

org.apache.accumulo.core.client.mock.MockConnectorTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.