Examples of org.apache.accumulo.core.client.BatchWriter

org.apache.accumulo.core.client.BatchWriter
Send Mutations to a single Table in Accumulo. When the user uses a Connector to create a BatchWriter, they specify how much memory and how many threads it should use. As the user adds mutations to the batch writer, it buffers them. Once the buffered mutations have used half of the user specified buffer, the mutations are dumped into the background to be written by a thread pool. If the user specified memory completely fills up, then writes are held. When a user calls flush, it does not return until all buffered mutations are written.

      return Collections.singleton(mergeInfo);
    }
  }
  
  private static void update(Connector c, Mutation m) throws TableNotFoundException, MutationsRejectedException {
    BatchWriter bw = c.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
    bw.addMutation(m);
    bw.close();
  }

View Full Code Here

    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(GCLotsOfCandidatesTest.class.getName(), args, bwOpts);
    
    Connector conn = opts.getConnector();
    conn.securityOperations().grantTablePermission(conn.whoami(), Constants.METADATA_TABLE_NAME, TablePermission.WRITE);
    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, bwOpts.getBatchWriterConfig());
    
    for (int i = 0; i < 100000; ++i) {
      final Text emptyText = new Text("");
      Text row = new Text(String.format("%s/%020d/%s", Constants.METADATA_DELETE_FLAG_PREFIX, i,
          "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj"));
      Mutation delFlag = new Mutation(row);
      delFlag.put(emptyText, emptyText, new Value(new byte[] {}));
      bw.addMutation(delFlag);
    }
    bw.close();
  }

View Full Code Here

  
  @Test
  public void test() throws Exception {
    Instance instance = new MockInstance();
    Connector connector = instance.getConnector("root", new PasswordToken(""));
    BatchWriter bw = connector.createBatchWriter("!METADATA", new BatchWriterConfig());
    
    // Create a fake METADATA table with these splits
    String splits[] = {"a", "e", "j", "o", "t", "z"};
    // create metadata for a table "t" with the splits above
    Text tableId = new Text("t");
    Text pr = null;
    for (String s : splits) {
      Text split = new Text(s);
      Mutation prevRow = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, split, pr));
      prevRow.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
      Constants.METADATA_CHOPPED_COLUMN.put(prevRow, new Value("junk".getBytes()));
      bw.addMutation(prevRow);
      pr = split;
    }
    // Add the default tablet
    Mutation defaultTablet = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, null, pr));
    defaultTablet.put(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
    bw.addMutation(defaultTablet);
    bw.close();
    
    // Read out the TabletLocationStates
    MockCurrentState state = new MockCurrentState(new MergeInfo(new KeyExtent(tableId, new Text("p"), new Text("e")), MergeInfo.Operation.MERGE));
    TCredentials auths = CredentialHelper.create("root", new PasswordToken(new byte[0]), "instance");

View Full Code Here


    Instance instance = new MockInstance();
    Connector conn = instance.getConnector("root", new PasswordToken(new byte[0]));


    conn.tableOperations().create(table);
    BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
    Mutation m = new Mutation(ball);
    m.put(new byte[0], new byte[0], new byte[0]);
    bw.addMutation(m);
    bw.close();


    IteratorSetting is = new IteratorSetting(5, RegExFilter.class);
    RegExFilter.setRegexs(is, s2, null, null, null, true, true);


    Scanner scanner = conn.createScanner(table, new Authorizations());

View Full Code Here

      
      final Text CF = new Text("cf"), CQ = new Text("cq");
      final byte[] CF_BYTES = "cf".getBytes(Constants.UTF8), CQ_BYTES = "cq".getBytes(Constants.UTF8);
      
      if (opts.mode.equals("ingest") || opts.mode.equals("delete")) {
        BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
        boolean delete = opts.mode.equals("delete");
        
        for (long i = 0; i < opts.num; i++) {
          byte[] row = encodeLong(i + opts.start);
          String value = "" + (i + opts.start);
          
          Mutation m = new Mutation(new Text(row));
          if (delete) {
            m.putDelete(CF, CQ);
          } else {
            m.put(CF, CQ, new Value(value.getBytes(Constants.UTF8)));
          }
          bw.addMutation(m);
        }
        
        bw.close();
      } else if (opts.mode.equals("verifyDeleted")) {
        Scanner s = connector.createScanner(opts.tableName, opts.auths);
        s.setBatchSize(scanOpts.scanBatchSize);
        Key startKey = new Key(encodeLong(opts.start), CF_BYTES, CQ_BYTES, new byte[0], Long.MAX_VALUE);
        Key stopKey = new Key(encodeLong(opts.start + opts.num - 1), CF_BYTES, CQ_BYTES, new byte[0], 0);

View Full Code Here

      }
      
      // presplit
      connector.tableOperations().create(opts.getTableName());
      connector.tableOperations().addSplits(opts.getTableName(), keys);
      BatchWriter b = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
      
      // populate
      for (int i = 0; i < opts.count; i++) {
        Mutation m = new Mutation(new Text(String.format("%05d", i)));
        m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes(Constants.UTF8)));
        b.addMutation(m);
      }
      b.close();
    }
    
    readBack(connector, opts, scanOpts);
    opts.stopTracing();
  }

View Full Code Here

    return trow;
  }
  
  private void insertData(String table) throws Exception {
    
    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
    
    for (int i = 0; i < ROW_LIMIT; i++) {
      Mutation m = new Mutation(createRow(i));
      
      for (int j = 0; j < CF_LIMIT; j++) {
        for (int k = 0; k < CQ_LIMIT; k++) {
          for (int t = 0; t < TS_LIMIT; t++) {
            m.put(createCF(j), createCQ(k), t, new Value(String.format("%06d_%03d_%03d_%03d", i, j, k, t).getBytes(Constants.UTF8)));
          }
        }
      }
      
      bw.addMutation(m);
    }
    
    bw.close();
  }

View Full Code Here

    m.putDelete(new Text(cf), new Text(cq), le);
  }
  
  private void insertData() throws Exception {
    
    BatchWriter bw = getConnector().createBatchWriter("vt", new BatchWriterConfig());
    Mutation m1 = new Mutation(new Text("row1"));
    
    mput(m1, "cf1", "cq1", "", "v1");
    mput(m1, "cf1", "cq1", "A", "v2");
    mput(m1, "cf1", "cq1", "B", "v3");
    mput(m1, "cf1", "cq1", "A&B", "v4");
    mput(m1, "cf1", "cq1", "A&(L|M)", "v5");
    mput(m1, "cf1", "cq1", "B&(L|M)", "v6");
    mput(m1, "cf1", "cq1", "A&B&(L|M)", "v7");
    mput(m1, "cf1", "cq1", "A&B&(L)", "v8");
    mput(m1, "cf1", "cq1", "A&FOO", "v9");
    mput(m1, "cf1", "cq1", "A&FOO&(L|M)", "v10");
    mput(m1, "cf1", "cq1", "FOO", "v11");
    mput(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)", "v12");
    mput(m1, "cf1", "cq1", "A&B&(L|M|FOO)", "v13");
    
    bw.addMutation(m1);
    bw.close();
  }

View Full Code Here

    bw.close();
  }
  
  private void deleteData() throws Exception {
    
    BatchWriter bw = getConnector().createBatchWriter("vt", new BatchWriterConfig());
    Mutation m1 = new Mutation(new Text("row1"));
    
    mputDelete(m1, "cf1", "cq1", "");
    mputDelete(m1, "cf1", "cq1", "A");
    mputDelete(m1, "cf1", "cq1", "A&B");
    mputDelete(m1, "cf1", "cq1", "B&(L|M)");
    mputDelete(m1, "cf1", "cq1", "A&B&(L)");
    mputDelete(m1, "cf1", "cq1", "A&FOO&(L|M)");
    mputDelete(m1, "cf1", "cq1", "(A|B)&FOO&(L|M)");
    mputDelete(m1, "cf1", "cq1", "FOO&A"); // should not delete anything
    
    bw.addMutation(m1);
    bw.close();
    
    Map<Set<String>,Set<String>> expected = new HashMap<Set<String>,Set<String>>();
    
    expected.put(nss("A", "L"), nss("v5"));
    expected.put(nss("A", "M"), nss("v5"));

View Full Code Here

    
    queryData(nss("A", "B", "FOO", "L", "M", "Z"), nss("A", "B", "FOO", "L", "M", "Z"), expected);
  }
  
  private void insertDefaultData() throws Exception {
    BatchWriter bw = getConnector().createBatchWriter("vt2", new BatchWriterConfig());
    Mutation m1 = new Mutation(new Text("row1"));
    
    mput(m1, "cf1", "cq1", "BASE", "v1");
    mput(m1, "cf1", "cq2", "DEFLABEL", "v2");
    mput(m1, "cf1", "cq3", "", "v3");
    
    bw.addMutation(m1);
    bw.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

clojure_accumulo.iterators.ClojureCombinerTest

clojure_accumulo.iterators.ClojureFilterTest

clojure_accumulo.iterators.ClojureMapperTest

org.apache.accumulo.core.client.admin.FindMaxTest

org.apache.accumulo.core.client.ClientSideIteratorTest

org.apache.accumulo.core.client.impl.TabletServerBatchDeleter

org.apache.accumulo.core.client.mapreduce.AccumuloInputFormatTest

org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat$AccumuloRecordWriter

org.apache.accumulo.core.client.mock.MockBatchDeleter

org.apache.accumulo.core.client.mock.MockConnectorTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.