Package org.apache.hadoop.hbase.regionserver.wal

Examples of org.apache.hadoop.hbase.regionserver.wal.WALEdit


  public void testLogRoll() throws Exception {
    long seq = 0;
    long baseline = 1000;
    long time = baseline;
    KeyValue kv = new KeyValue(r1, f1, r1);
    WALEdit edit = new WALEdit();
    edit.add(kv);

    List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
    listeners.add(replication);
    HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
      URLEncoder.encode("regionserver:60020", "UTF8"));
View Full Code Here


    long seenEntries = 0;
    this.repLogReader.seek();
    HLog.Entry entry =
        this.repLogReader.readNextAndSetPosition();
    while (entry != null) {
      WALEdit edit = entry.getEdit();
      this.metrics.logEditsReadRate.inc(1);
      seenEntries++;
      // Remove all KVs that should not be replicated
      HLogKey logKey = entry.getKey();
      List<UUID> consumedClusterIds = edit.getClusterIds();
      // This cluster id has been added to resolve the scenario of A -> B -> A where A has old
      // point release and B has the new point release which has the fix HBASE-7709. A change on
      // cluster A would infinitely replicate to
      // cluster B if we don't add the original cluster id to the set.
      consumedClusterIds.add(logKey.getClusterId());
      // don't replicate if the log entries if it has not already been replicated
      if (!consumedClusterIds.contains(peerClusterId)) {
        removeNonReplicableEdits(edit);
        // Don't replicate catalog entries, if the WALEdit wasn't
        // containing anything to replicate and if we're currently not set to replicate
        if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) ||
            Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) &&
            edit.size() != 0 && replicating.get()) {
          // Only set the clusterId if is a local key.
          // This ensures that the originator sets the cluster id
          // and all replicas retain the initial cluster id.
          // This is *only* place where a cluster id other than the default is set.
          if (HConstants.DEFAULT_CLUSTER_ID == logKey.getClusterId()) {
            logKey.setClusterId(this.clusterId);
          } else if (logKey.getClusterId() != this.clusterId) {
            edit.addClusterId(clusterId);
          }
          currentNbOperations += countDistinctRowKeys(edit);
          entries.add(entry);
          currentSize += entry.getEdit().heapSize();
        } else {
View Full Code Here

      // flushCommits once per
      // invocation of this method per table and clusters that have consumed the change.
      Map<byte[], Map<List<UUID>, List<Row>>> rowMap =
          new TreeMap<byte[], Map<List<UUID>, List<Row>>>(Bytes.BYTES_COMPARATOR);
      for (HLog.Entry entry : entries) {
        WALEdit edit = entry.getEdit();
        byte[] table = entry.getKey().getTablename();
        Put put = null;
        Delete del = null;
        KeyValue lastKV = null;
        List<KeyValue> kvs = edit.getKeyValues();
        for (KeyValue kv : kvs) {
          if (lastKV == null || lastKV.getType() != kv.getType() || !lastKV.matchingRow(kv)) {
            UUID clusterId = entry.getKey().getClusterId();
            List<UUID> clusterIds = edit.getClusterIds();
            if (kv.isDelete()) {
              del = new Delete(kv.getRow());
              del.setClusterId(clusterId);
              del.setClusterIds(clusterIds);
              clusterIds.add(clusterId);
View Full Code Here

    long seenEntries = 0;
    this.repLogReader.seek();
    HLog.Entry entry =
        this.repLogReader.readNextAndSetPosition();
    while (entry != null) {
      WALEdit edit = entry.getEdit();
      this.metrics.logEditsReadRate.inc(1);
      seenEntries++;
      // Remove all KVs that should not be replicated
      HLogKey logKey = entry.getKey();
      List<UUID> consumedClusterIds = edit.getClusterIds();
      // This cluster id has been added to resolve the scenario of A -> B -> A where A has old
      // point release and B has the new point release which has the fix HBASE-7709. A change on
      // cluster A would infinitely replicate to
      // cluster B if we don't add the original cluster id to the set.
      consumedClusterIds.add(logKey.getClusterId());
      // don't replicate if the log entries if it has not already been replicated
      if (!consumedClusterIds.contains(peerClusterId)) {
        removeNonReplicableEdits(edit);
        // Don't replicate catalog entries, if the WALEdit wasn't
        // containing anything to replicate and if we're currently not set to replicate
        if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) ||
            Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) &&
            edit.size() != 0 && replicating.get()) {
          // Only set the clusterId if is a local key.
          // This ensures that the originator sets the cluster id
          // and all replicas retain the initial cluster id.
          // This is *only* place where a cluster id other than the default is set.
          if (HConstants.DEFAULT_CLUSTER_ID == logKey.getClusterId()) {
            logKey.setClusterId(this.clusterId);
          } else if (logKey.getClusterId() != this.clusterId) {
            edit.addClusterId(clusterId);
          }
          currentNbOperations += countDistinctRowKeys(edit);
          entries.add(entry);
          currentSize += entry.getEdit().heapSize();
        } else {
View Full Code Here

        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
        fs.create(recoveredEdits);
        HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);

        long time = System.nanoTime();
        WALEdit edit = new WALEdit();
        edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
            .toBytes(i)));
        writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
            HConstants.DEFAULT_CLUSTER_ID), edit));

        writer.close();
View Full Code Here

        Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
        fs.create(recoveredEdits);
        HLog.Writer writer = HLogFactory.createWriter(fs, recoveredEdits, conf);

        long time = System.nanoTime();
        WALEdit edit = new WALEdit();
        edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes
            .toBytes(i)));
        writer.append(new HLog.Entry(new HLogKey(regionName, tableName, i, time,
            HConstants.DEFAULT_CLUSTER_ID), edit));

        writer.close();
View Full Code Here

    }
    int n = hris.size();
    int[] counts = new int[n];
    if (n > 0) {
      for (int i = 0; i < num_edits; i += 1) {
        WALEdit e = new WALEdit();
        HRegionInfo curRegionInfo = hris.get(i % n);
        byte[] startRow = curRegionInfo.getStartKey();
        if (startRow == null || startRow.length == 0) {
          startRow = new byte[] { 0, 0, 0, 0, 1 };
        }
        byte[] row = Bytes.incrementBytes(startRow, counts[i % n]);
        row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because
                                             // HBaseTestingUtility.createMultiRegions use 5 bytes
                                             // key
        byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
        e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
        log.append(curRegionInfo, fullTName, e, System.currentTimeMillis(), htd);
        counts[i % n] += 1;
      }
    }
    log.sync();
View Full Code Here

    this.repLogReader.seek();
    long positionBeforeRead = this.repLogReader.getPosition();
    HLog.Entry entry =
        this.repLogReader.readNextAndSetPosition();
    while (entry != null) {
      WALEdit edit = entry.getEdit();
      this.metrics.incrLogEditsRead();
      seenEntries++;
      // Remove all KVs that should not be replicated
      HLogKey logKey = entry.getKey();
      // don't replicate if the log entries have already been consumed by the cluster
      if (!logKey.getClusterIds().contains(peerClusterId)) {
        removeNonReplicableEdits(entry);
        // Don't replicate catalog entries, if the WALEdit wasn't
        // containing anything to replicate and if we're currently not set to replicate
        if (!logKey.getTablename().equals(TableName.META_TABLE_NAME) &&
            edit.size() != 0) {
          //Mark that the current cluster has the change
          logKey.addClusterId(clusterId);
          currentNbOperations += countDistinctRowKeys(edit);
          entries.add(entry);
          currentSize += entry.getEdit().heapSize();
View Full Code Here

          byte[] regionName = getRegionName(tableName, j);
          for (int k = 0; k < 50; ++k) {
            byte[] rowkey = Bytes.toBytes("row-" + k);
            HLogKey key = new HLogKey(regionName, tableName, (long)k,
              System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
            WALEdit edit = new WALEdit();
            edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey));
            writer.append(new HLog.Entry(key, edit));
          }
        }
      }
    } finally {
View Full Code Here


  private void doPreMutationHook(BatchOperationInProgress<?> batchOp)
      throws IOException {
    /* Run coprocessor pre hook outside of locks to avoid deadlock */
    WALEdit walEdit = new WALEdit();
    if (coprocessorHost != null) {
      for (int i = 0 ; i < batchOp.operations.length; i++) {
        Mutation m = batchOp.getMutation(i);
        if (m instanceof Put) {
          if (coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) {
            // pre hook says skip this Put
            // mark as success and skip in doMiniBatchMutation
            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
          }
        } else if (m instanceof Delete) {
          Delete curDel = (Delete) m;
          if (curDel.getFamilyCellMap().isEmpty()) {
            // handle deleting a row case
            prepareDelete(curDel);
          }
          if (coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) {
            // pre hook says skip this Delete
            // mark as success and skip in doMiniBatchMutation
            batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
          }
        } else {
          // In case of passing Append mutations along with the Puts and Deletes in batchMutate
          // mark the operation return code as failure so that it will not be considered in
          // the doMiniBatchMutation
          batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.FAILURE,
              "Put/Delete mutations only supported in batchMutate() now");
        }
        if (!walEdit.isEmpty()) {
          batchOp.walEditsFromCoprocessors[i] = walEdit;
          walEdit = new WALEdit();
        }
      }
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.wal.WALEdit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.