Package org.apache.accumulo.core.client

Examples of org.apache.accumulo.core.client.BatchWriter

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    long maxMemory = Long.parseLong(getTestProperty("MAX_MEMORY"));
    long maxLatency = Long.parseLong(getTestProperty("MAX_LATENCY"));
    int maxWriteThreads = Integer.parseInt(getTestProperty("NUM_THREADS"));
   
    // create batch writer
    BatchWriter bw = null;
    try {
      bw = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
          .setMaxWriteThreads(maxWriteThreads));
    } catch (TableNotFoundException e) {
      System.out.println("Table not found: " + tableName);
      e.printStackTrace();
    }
   
    // configure writing
    Random r = new Random();
    String ingestInstanceId = UUID.randomUUID().toString();
    long numIngestEntries = Long.parseLong(getTestProperty("NUM_ENTRIES"));
    long minRow = 0L;
    long maxRow = 9223372036854775807L;
    int maxColF = 32767;
    int maxColQ = 32767;
    long count = 0;
    long totalBytes = 0;
   
    ColumnVisibility cv = new ColumnVisibility();

    // start timer
    startTimer();
   
    // write specified number of entries
    while (count < numIngestEntries) {
      count++;
      long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
      Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId.getBytes(Constants.UTF8), count, null, r, false);
      totalBytes += m.numBytes();
      try {
        bw.addMutation(m);
      } catch (MutationsRejectedException e) {
        e.printStackTrace();
        System.exit(-1);
      }
    }
   
    // close writer
    try {
      bw.close();
    } catch (MutationsRejectedException e) {
      e.printStackTrace();
      System.exit(-1);
    }
   
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    Text cf = new Text("cf1");
    Text cq = new Text("cq1");
   
    getConnector().tableOperations().create("t1");
    getConnector().tableOperations().addSplits("t1", splits);
    BatchWriter bw = getConnector().createBatchWriter("t1", new BatchWriterConfig());
   
    for (int i = 1; i < 257; i++) {
      Mutation m = new Mutation(new Text(String.format("%08x", (i << 8) - 16)));
      m.put(cf, cq, new Value(Integer.toString(i).getBytes(Constants.UTF8)));
     
      bw.addMutation(m);
    }
   
    bw.close();
   
    // verify data is there
    Scanner scanner1 = getConnector().createScanner("t1", Constants.NO_AUTHS);
   
    int ei = 1;
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
   
    extents.add(new KeyExtent(tid, null, per));
   
    if (args[0].equals("write")) {
     
      BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
     
      for (KeyExtent extent : extents) {
        Mutation mut = extent.getPrevRowUpdateMutation();
        new TServerInstance(AddressUtil.parseAddress("192.168.1.100", 4567), "DEADBEEF").putLocation(mut);
        bw.addMutation(mut);
      }
     
      bw.close();
    } else if (args[0].equals("writeFiles")) {
      BatchWriter bw = connector.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
     
      for (KeyExtent extent : extents) {
       
        Mutation mut = new Mutation(extent.getMetadataEntry());
       
        String dir = "/t-" + UUID.randomUUID();
       
        Constants.METADATA_DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(Constants.UTF8)));
       
        for (int i = 0; i < 5; i++) {
          mut.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(Constants.UTF8)));
        }
       
        bw.addMutation(mut);
      }
     
      bw.close();
    } else if (args[0].equals("scan")) {
     
      int numThreads = Integer.parseInt(args[1]);
      final int numLoop = Integer.parseInt(args[2]);
      int numLookups = Integer.parseInt(args[3]);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
    throw new UnsupportedOperationException("Cannot filter on value with deleter; Write your own deleter");
  }

  @Override
  public void delete() throws MutationsRejectedException, TableNotFoundException {
    BatchWriter bw = null;
    try {
      bw = new BatchWriterImpl(instance, credentials, tableId, maxMemory, maxLatency, maxWriteThreads);
      Iterator<Entry<Key,Value>> iter = super.iterator();
      while (iter.hasNext()) {
        Entry<Key,Value> next = iter.next();
        Key k = next.getKey();
        Mutation m = new Mutation(k.getRow());
        m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp());
        bw.addMutation(m);
      }
    } finally {
      if (bw != null)
        bw.close();
    }
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  @Test
  public void test() throws Exception {
    MockInstance instance = new MockInstance("instance1");
    Connector conn = instance.getConnector("root", "".getBytes());
    conn.tableOperations().create("test");
    BatchWriter bw = conn.createBatchWriter("test", 100000l, 100l, 5);
   
    insertList(bw, row1);
    insertList(bw, row2);
    insertList(bw, row3);
    bw.close();
   
    JobContext job = ContextFactory.createJobContext();
    AccumuloRowInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations());
    AccumuloRowInputFormat.setMockInstance(job.getConfiguration(), "instance1");
    AccumuloRowInputFormat crif = new AccumuloRowInputFormat();
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  public void testMR() throws Exception {
    MockInstance mockInstance = new MockInstance("testmrinstance");
    Connector c = mockInstance.getConnector("root", new byte[] {});
    c.tableOperations().create("testtable1");
    c.tableOperations().create("testtable2");
    BatchWriter bw = c.createBatchWriter("testtable1", 10000L, 1000L, 4);
    for (int i = 0; i < 100; i++) {
      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();

    Job job = new Job();
    job.setInputFormatClass(AccumuloInputFormat.class);
    job.setMapperClass(TestMapper.class);
    job.setOutputFormatClass(AccumuloOutputFormat.class);
View Full Code Here
432
433
434
435
436
437
438
439
440
441
442
        log.info("Simulating adding table: " + tableName);
        return;
      }
     
      log.debug("Adding table: " + tableName);
      BatchWriter bw = null;
      String table = tableName.toString();
     
      if (createTables && !conn.tableOperations().exists(table)) {
        try {
          conn.tableOperations().create(table);
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
187
    Connector conn = new ZooKeeperInstance(instance, zooKeepers).getConnector(user, pass.getBytes());
    if (!conn.tableOperations().exists(dataTable)) {
      conn.tableOperations().create(dataTable);
      conn.tableOperations().attachIterator(dataTable, new IteratorSetting(1, ChunkCombiner.class));
    }
    BatchWriter bw = conn.createBatchWriter(dataTable, 50000000, 300000l, 4);
    FileDataIngest fdi = new FileDataIngest(chunkSize, colvis);
    for (int i = 7; i < args.length; i++) {
      fdi.insertFileData(args[i], bw);
    }
    bw.close();
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
  @Test
  public void testSunnyDay() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", 10000L, 1000L, 4);
    for (int i = 0; i < 100; i++) {
      int r = random.nextInt();
      Mutation m = new Mutation(asText(r));
      m.put(asText(random.nextInt()), asText(random.nextInt()), new Value(Integer.toHexString(r).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    BatchScanner s = c.createBatchScanner("test", Constants.NO_AUTHS, 2);
    s.setRanges(Collections.singletonList(new Range()));
    Key key = null;
    int count = 0;
    for (Entry<Key,Value> entry : s) {
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
  @Test
  public void testBadMutations() throws Exception {
    Connector c = new MockConnector("root", new MockInstance());
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", 10000L, 1000L, 4);

    try {
      bw.addMutation(null);
      Assert.fail("addMutation should throw IAE for null mutation");
    } catch (IllegalArgumentException iae) {}
    try {
      bw.addMutations(null);
      Assert.fail("addMutations should throw IAE for null iterable");
    } catch (IllegalArgumentException iae) {}

    bw.addMutations(Collections.<Mutation>emptyList());

    Mutation bad = new Mutation("bad");
    try {
      bw.addMutation(bad);
      Assert.fail("addMutation should throw IAE for empty mutation");
    } catch (IllegalArgumentException iae) {}


    Mutation good = new Mutation("good");
    good.put(asText(random.nextInt()), asText(random.nextInt()), new Value("good".getBytes()));
    List<Mutation> mutations = new ArrayList<Mutation>();
    mutations.add(good);
    mutations.add(bad);
    try {
      bw.addMutations(mutations);
      Assert.fail("addMutations should throw IAE if it contains empty mutation");
    } catch (IllegalArgumentException iae) {}

    bw.close();
  }
View Full Code Here
TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

  • clojure_accumulo.iterators.ClojureCombinerTest
  • clojure_accumulo.iterators.ClojureFilterTest
  • clojure_accumulo.iterators.ClojureMapperTest
  • org.apache.accumulo.core.client.admin.FindMaxTest
  • org.apache.accumulo.core.client.ClientSideIteratorTest
  • org.apache.accumulo.core.client.impl.TabletServerBatchDeleter
  • org.apache.accumulo.core.client.mapreduce.AccumuloInputFormatTest
  • org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat$AccumuloRecordWriter
  • org.apache.accumulo.core.client.mock.MockBatchDeleter
  • org.apache.accumulo.core.client.mock.MockConnectorTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.