Package org.apache.accumulo.core.client

Examples of org.apache.accumulo.core.client.BatchWriter


      byte randomValue[] = new byte[opts.dataSize];
      Random random = new Random();
     
      long bytesWritten = 0;
     
      BatchWriter bw = null;
      FileSKVWriter writer = null;
     
      if (opts.outputFile != null) {
        Configuration conf = CachedConfiguration.getInstance();
        FileSystem fs = FileSystem.get(conf);
        writer = FileOperations.getInstance().openWriter(opts.outputFile + "." + RFile.EXTENSION, fs, conf,
            AccumuloConfiguration.getDefaultConfiguration());
        writer.startDefaultLocalityGroup();
      } else {
        Connector connector = opts.getConnector();
        bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
      }
      Text labBA = new Text(opts.columnVisibility.getExpression());
     
      long startTime = System.currentTimeMillis();
      for (int i = 0; i < opts.rows; i++) {
        int rowid;
        if (opts.stride > 0) {
          rowid = ((i % opts.stride) * (opts.rows / opts.stride)) + (i / opts.stride);
        } else {
          rowid = i;
        }
       
        Text row = generateRow(rowid, opts.startRow);
        Mutation m = new Mutation(row);
        for (int j = 0; j < opts.cols; j++) {
          Text colf = new Text(opts.columnFamily);
          Text colq = new Text(FastFormat.toZeroPaddedString(j, 7, 10, COL_PREFIX));
         
          if (writer != null) {
            Key key = new Key(row, colf, colq, labBA);
            if (opts.timestamp >= 0) {
              key.setTimestamp(opts.timestamp);
            } else {
              key.setTimestamp(startTime);
            }
           
            if (opts.delete) {
              key.setDeleted(true);
            } else {
              key.setDeleted(false);
            }
           
            bytesWritten += key.getSize();
           
            if (opts.delete) {
              writer.append(key, new Value(new byte[0]));
            } else {
              byte value[];
              if (opts.random != null) {
                value = genRandomValue(random, randomValue, opts.random.intValue(), rowid + opts.startRow, j);
              } else {
                value = bytevals[j % bytevals.length];
              }
             
              Value v = new Value(value);
              writer.append(key, v);
              bytesWritten += v.getSize();
            }
           
          } else {
            Key key = new Key(row, colf, colq, labBA);
            bytesWritten += key.getSize();
           
            if (opts.delete) {
              if (opts.timestamp >= 0)
                m.putDelete(colf, colq, opts.columnVisibility, opts.timestamp);
              else
                m.putDelete(colf, colq, opts.columnVisibility);
            } else {
              byte value[];
              if (opts.random != null) {
                value = genRandomValue(random, randomValue, opts.random.intValue(), rowid + opts.startRow, j);
              } else {
                value = bytevals[j % bytevals.length];
              }
              bytesWritten += value.length;
             
              if (opts.timestamp >= 0) {
                m.put(colf, colq, opts.columnVisibility, opts.timestamp, new Value(value, true));
              } else {
                m.put(colf, colq, opts.columnVisibility, new Value(value, true));
               
              }
            }
          }
         
        }
        if (bw != null)
          bw.addMutation(m);
       
      }
     
      if (writer != null) {
        writer.close();
      } else if (bw != null) {
        try {
          bw.close();
        } catch (MutationsRejectedException e) {
          if (e.getAuthorizationFailuresMap().size() > 0) {
            for (Entry<KeyExtent,Set<SecurityErrorCode>> entry : e.getAuthorizationFailuresMap().entrySet()) {
              System.err.println("ERROR : Not authorized to write to : " + entry.getKey() + " due to " + entry.getValue());
            }
View Full Code Here


   
    conn.tableOperations().attachIterator("table1", is);
   
    Connector uconn = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()).getConnector("user1", new PasswordToken("pass1"));
   
    BatchWriter bw = uconn.createBatchWriter("table1", new BatchWriterConfig());
   
    UUID uuid = UUID.randomUUID();
   
    Mutation m = new Mutation(uuid.toString());
    m.put("META", "SIZE", new ColumnVisibility("A|B"), "8");
    m.put("META", "CRC", new ColumnVisibility("A|B"), "456");
    m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
    m.put("DATA", "IMG", new ColumnVisibility("A&B"), "ABCDEFGH");
   
    bw.addMutation(m);
    bw.flush();
   
    m = new Mutation(uuid.toString());
    m.put("META", "COUNT", new ColumnVisibility("A|B"), "1");
    m.put("META", "CRC", new ColumnVisibility("A|B"), "123");
    bw.addMutation(m);
   
    bw.close();
   
    int count = 0;
    Scanner scanner = uconn.createScanner("table1", new Authorizations("A"));
    for (Entry<Key,Value> entry : scanner) {
      if (entry.getKey().getColumnQualifierData().toString().equals("COUNT")) {
View Full Code Here

  }
 
  public static void deleteTable(String tableId, boolean insertDeletes, TCredentials credentials, ZooLock lock) throws AccumuloException {
    Scanner ms = new ScannerImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
    Text tableIdText = new Text(tableId);
    BatchWriter bw = new BatchWriterImpl(HdfsZooInstance.getInstance(), credentials, Constants.METADATA_TABLE_ID, new BatchWriterConfig().setMaxMemory(1000000)
        .setMaxLatency(120000l, TimeUnit.MILLISECONDS).setMaxWriteThreads(2));
   
    // scan metadata for our table and delete everything we find
    Mutation m = null;
    ms.setRange(new KeyExtent(tableIdText, null, null).toMetadataRange());

    // insert deletes before deleting data from !METADATA... this makes the code fault tolerant
    if (insertDeletes) {

      ms.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
      Constants.METADATA_DIRECTORY_COLUMN.fetch(ms);
     
      for (Entry<Key,Value> cell : ms) {
        Key key = cell.getKey();

        if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
          String relPath = key.getColumnQualifier().toString();
          // only insert deletes for files owned by this table
          if (!relPath.startsWith("../"))
            bw.addMutation(createDeleteMutation(tableId, relPath));
        }

        if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
          bw.addMutation(createDeleteMutation(tableId, cell.getValue().toString()));
        }
      }

      bw.flush();

      ms.clearColumns();
    }

    for (Entry<Key,Value> cell : ms) {
      Key key = cell.getKey();

      if (m == null) {
        m = new Mutation(key.getRow());
        if (lock != null)
          putLockID(lock, m);
      }

      if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) {
        bw.addMutation(m);
        m = new Mutation(key.getRow());
        if (lock != null)
          putLockID(lock, m);
      }
      m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
    }

    if (m != null)
      bw.addMutation(m);

    bw.close();
  }
View Full Code Here

   
    conn.instanceOperations().setProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1", jarFile.toURI().toString());
    conn.tableOperations().setProperty("table2", Property.TABLE_CLASSPATH.getKey(), "cx1");
    conn.tableOperations().attachIterator("table2", new IteratorSetting(100, "foocensor", "org.apache.accumulo.test.FooFilter"));
   
    BatchWriter bw = conn.createBatchWriter("table2", new BatchWriterConfig());
   
    Mutation m1 = new Mutation("foo");
    m1.put("cf1", "cq1", "v2");
    m1.put("cf1", "cq2", "v3");
   
    bw.addMutation(m1);
   
    Mutation m2 = new Mutation("bar");
    m2.put("cf1", "cq1", "v6");
    m2.put("cf1", "cq2", "v7");
   
    bw.addMutation(m2);
   
    bw.close();
   
    Scanner scanner = conn.createScanner("table2", new Authorizations());
   
    int count = 0;
    for (Entry<Key,Value> entry : scanner) {
View Full Code Here

  }

  public static void cloneTable(Instance instance, String srcTableId, String tableId) throws Exception {
   
    Connector conn = instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());
    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
   
    while (true) {

      try {
        initializeClone(srcTableId, tableId, conn, bw);

        // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed

        while (true) {
          int rewrites = checkClone(srcTableId, tableId, conn, bw);

          if (rewrites == 0)
            break;
        }

        bw.flush();
        break;

      } catch (TabletIterator.TabletDeletedException tde) {
        // tablets were merged in the src table
        bw.flush();

        // delete what we have cloned and try again
        deleteTable(tableId, false, SecurityConstants.getSystemCredentials(), null);

        log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again");

        UtilWaitThread.sleep(100);
      }
    }

    // delete the clone markers and create directory entries
    Scanner mscanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
    mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY);

    int dirCount = 0;

    for (Entry<Key,Value> entry : mscanner) {
      Key k = entry.getKey();
      Mutation m = new Mutation(k.getRow());
      m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
      Constants.METADATA_DIRECTORY_COLUMN.put(m, new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes(Constants.UTF8))));
      bw.addMutation(m);
    }

    bw.close();

  }
View Full Code Here

  public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception {
    Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
    mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY);
    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
    for (Entry<Key,Value> entry : mscanner) {
      log.debug("Looking at entry " + entry + " with tid " + tid);
      if (Long.parseLong(entry.getValue().toString()) == tid) {
        log.debug("deleting entry " + entry);
        Mutation m = new Mutation(entry.getKey().getRow());
        m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
        bw.addMutation(m);
      }
    }
    bw.close();
  }
View Full Code Here

    Connector c = inst.getConnector("root", new PasswordToken(passwd));
   
    final String table = "foobar";
    c.tableOperations().create(table);
   
    BatchWriter bw = null;
   
    // Add some data
    try {
      bw = c.createBatchWriter(table, new BatchWriterConfig());
      Mutation m = new Mutation("a");
      for (int i = 0; i < 50; i++) {
        m.put("colf", Integer.toString(i), "");
      }
     
      bw.addMutation(m);
    } finally {
      if (null != bw) {
        bw.close();
      }
    }
   
    final boolean flush = true, wait = true;
   
View Full Code Here

    }
  }
 
  protected void writeVersionable(Connector c, String tableName, int size) throws TableNotFoundException, MutationsRejectedException {
    for (int i = 0; i < size; i++) {
      BatchWriter w = c.createBatchWriter(tableName, new BatchWriterConfig());
      Mutation m = new Mutation("row1");
      m.put("cf", "cq", String.valueOf(i));
      w.addMutation(m);
      w.close();
    }
  }
View Full Code Here

  @Test
  public void testMap() throws Exception {
    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
    Connector c = mockInstance.getConnector("root", new PasswordToken(""));
    c.tableOperations().create(TEST_TABLE_1);
    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
    for (int i = 0; i < 100; i++) {
      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
   
    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
    assertNull(e1);
    assertNull(e2);
  }
View Full Code Here

  public void testDeleteRows() throws Exception {
    Instance instance = new MockInstance("rows");
    Connector connector = instance.getConnector("user", new PasswordToken("foo".getBytes()));
    TableOperations to = connector.tableOperations();
    to.create("test");
    BatchWriter bw = connector.createBatchWriter("test", new BatchWriterConfig());
    for (int r = 0; r < 20; r++) {
      Mutation m = new Mutation("" + r);
      for (int c = 0; c < 5; c++) {
        m.put(new Text("cf"), new Text("" + c), new Value(("" + c).getBytes()));
      }
      bw.addMutation(m);
    }
    bw.flush();
    to.deleteRows("test", new Text("1"), new Text("2"));
    Scanner s = connector.createScanner("test", Constants.NO_AUTHS);
    int oneCnt = 0;
    for (Entry<Key,Value> entry : s) {
      char rowStart = entry.getKey().getRow().toString().charAt(0);
View Full Code Here

TOP

Related Classes of org.apache.accumulo.core.client.BatchWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.