Examples of org.apache.hadoop.hbase.regionserver.wal.HLogKey

Package org.apache.hadoop.hbase.regionserver.wal

Examples of org.apache.hadoop.hbase.regionserver.wal.HLogKey

org.apache.hadoop.hbase.regionserver.wal.HLogKey
A Key for an entry in the change log. The log intermingles edits to many tables and rows, so each log entry identifies the appropriate table and row. Within a table and row, they're also sorted.
Some Transactional edits (START, COMMIT, ABORT) will not have an associated row.


        long time = System.nanoTime();
        WALEdit edit = new WALEdit();
        edit.add(new KeyValue(row, family, Bytes.toBytes(i),
            time, KeyValue.Type.Put, Bytes.toBytes(i)));
        writer.append(new HLog.Entry(new HLogKey(regionName, tableName,
            i, time, HConstants.DEFAULT_CLUSTER_ID), edit));


        writer.close();
      }
      long recoverSeqId = 1030;

View Full Code Here

    for(long i = 1; i < 101; i++) {
      if(i > 1 && i % 20 == 0) {
        hlog.rollWriter();
      }
      LOG.info(i);
      HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
          System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
      hlog.append(hri, key, edit, htd, true);
    }


    // Simulate a rapid insert that's followed
    // by a report that's still not totally complete (missing last one)
    LOG.info(baseline + " and " + time);
    baseline += 101;
    time = baseline;
    LOG.info(baseline + " and " + time);


    for (int i = 0; i < 3; i++) {
      HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
          System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
      hlog.append(hri, key, edit, htd, true);
    }


    assertEquals(6, manager.getHLogs().get(slaveId).size());


    hlog.rollWriter();


    manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
        "1", 0, false, false);


    HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
        System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
    hlog.append(hri, key, edit, htd, true);


    assertEquals(1, manager.getHLogs().size());

View Full Code Here

    for(int i = 0; i < 3; i++) {
      byte[] b = Bytes.toBytes(Integer.toString(i));
      KeyValue kv = new KeyValue(b,b,b);
      WALEdit edit = new WALEdit();
      edit.add(kv);
      HLogKey key = new HLogKey(b, b, 0, 0, HConstants.DEFAULT_CLUSTER_ID);
      writer.append(new HLog.Entry(key, edit));
      writer.sync();
    }
    writer.close();

View Full Code Here

          this.conf.getInt("hbase.master.assignment.timeoutmonitor.timeout",
              180000) / 2);
      long lastReport = EnvironmentEdgeManager.currentTimeMillis();


      while ((entry = reader.next()) != null) {
        HLogKey key = entry.getKey();
        WALEdit val = entry.getEdit();


        if (reporter != null) {
          intervalEdits += val.size();
          if (intervalEdits >= interval) {
            // Number of edits interval reached
            intervalEdits = 0;
            long cur = EnvironmentEdgeManager.currentTimeMillis();
            if (lastReport + period <= cur) {
              // Timeout reached
              if(!reporter.progress()) {
                String msg = "Progressable reporter failed, stopping replay";
                LOG.warn(msg);
                throw new IOException(msg);
              }
              lastReport = cur;
            }
          }
        }


        if (firstSeqIdInLog == -1) {
          firstSeqIdInLog = key.getLogSeqNum();
        }
        // Now, figure if we should skip this edit.
        if (key.getLogSeqNum() <= currentEditSeqId) {
          skippedEdits++;
          continue;
        }
        currentEditSeqId = key.getLogSeqNum();
        boolean flush = false;
        for (KeyValue kv: val.getKeyValues()) {
          // Check this edit is for me. Also, guard against writing the special
          // METACOLUMN info such as HBASE::CACHEFLUSH entries
          if (kv.matchingFamily(HLog.METAFAMILY) ||
              !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
            skippedEdits++;
            continue;
              }
          // Figure which store the edit is meant for.
          if (store == null || !kv.matchingFamily(store.getFamily().getName())) {

View Full Code Here

      WALEdit edit = entry.getEdit();
      this.metrics.logEditsReadRate.inc(1);
      seenEntries++;
      // Remove all KVs that should not be replicated
      removeNonReplicableEdits(edit);
      HLogKey logKey = entry.getKey();
      // Don't replicate catalog entries, if the WALEdit wasn't
      // containing anything to replicate and if we're currently not set to replicate
      if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) ||
          Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) &&
          edit.size() != 0 && replicating.get()) {
        logKey.setClusterId(this.clusterId);
        currentNbOperations += countDistinctRowKeys(edit);
        currentNbEntries++;
      } else {
        this.metrics.logEditsFilteredRate.inc(1);
      }

View Full Code Here

      WALEdit edit = entry.getEdit();
      this.metrics.logEditsReadRate.inc(1);
      seenEntries++;
      // Remove all KVs that should not be replicated
      removeNonReplicableEdits(edit);
      HLogKey logKey = entry.getKey();
      // Don't replicate catalog entries, if the WALEdit wasn't
      // containing anything to replicate and if we're currently not set to replicate
      if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) ||
          Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) &&
          edit.size() != 0 && replicating.get()) {
        logKey.setClusterId(this.clusterId);
        currentNbOperations += countDistinctRowKeys(edit);
        currentNbEntries++;
      } else {
        this.metrics.logEditsFilteredRate.inc(1);
      }

View Full Code Here

          this.conf.getInt("hbase.master.assignment.timeoutmonitor.timeout",
              180000) / 2);
      long lastReport = EnvironmentEdgeManager.currentTimeMillis();


      while ((entry = reader.next()) != null) {
        HLogKey key = entry.getKey();
        WALEdit val = entry.getEdit();


        if (reporter != null) {
          intervalEdits += val.size();
          if (intervalEdits >= interval) {
            // Number of edits interval reached
            intervalEdits = 0;
            long cur = EnvironmentEdgeManager.currentTimeMillis();
            if (lastReport + period <= cur) {
              // Timeout reached
              if(!reporter.progress()) {
                String msg = "Progressable reporter failed, stopping replay";
                LOG.warn(msg);
                throw new IOException(msg);
              }
              lastReport = cur;
            }
          }
        }


        if (firstSeqIdInLog == -1) {
          firstSeqIdInLog = key.getLogSeqNum();
        }
        // Now, figure if we should skip this edit.
        if (key.getLogSeqNum() <= currentEditSeqId) {
          skippedEdits++;
          continue;
        }
        currentEditSeqId = key.getLogSeqNum();
        boolean flush = false;
        for (KeyValue kv: val.getKeyValues()) {
          // Check this edit is for me. Also, guard against writing the special
          // METACOLUMN info such as HBASE::CACHEFLUSH entries
          if (kv.matchingFamily(HLog.METAFAMILY) ||
              !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
            skippedEdits++;
            continue;
              }
          // Figure which store the edit is meant for.
          if (store == null || !kv.matchingFamily(store.getFamily().getName())) {

View Full Code Here

          this.conf.getInt("hbase.master.assignment.timeoutmonitor.timeout",
              180000) / 2);
      long lastReport = EnvironmentEdgeManager.currentTimeMillis();


      while ((entry = reader.next()) != null) {
        HLogKey key = entry.getKey();
        WALEdit val = entry.getEdit();


        if (reporter != null) {
          intervalEdits += val.size();
          if (intervalEdits >= interval) {
            // Number of edits interval reached
            intervalEdits = 0;
            long cur = EnvironmentEdgeManager.currentTimeMillis();
            if (lastReport + period <= cur) {
              // Timeout reached
              if(!reporter.progress()) {
                String msg = "Progressable reporter failed, stopping replay";
                LOG.warn(msg);
                throw new IOException(msg);
              }
              lastReport = cur;
            }
          }
        }


        if (firstSeqIdInLog == -1) {
          firstSeqIdInLog = key.getLogSeqNum();
        }
        // Now, figure if we should skip this edit.
        if (key.getLogSeqNum() <= currentEditSeqId) {
          skippedEdits++;
          continue;
        }
        currentEditSeqId = key.getLogSeqNum();
        boolean flush = false;
        for (KeyValue kv: val.getKeyValues()) {
          // Check this edit is for me. Also, guard against writing the special
          // METACOLUMN info such as HBASE::CACHEFLUSH entries
          if (kv.matchingFamily(HLog.METAFAMILY) ||
              !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
            skippedEdits++;
            continue;
              }
          // Figure which store the edit is meant for.
          if (store == null || !kv.matchingFamily(store.getFamily().getName())) {

View Full Code Here

      WALEdit edit = entry.getEdit();
      this.metrics.logEditsReadRate.inc(1);
      seenEntries++;
      // Remove all KVs that should not be replicated
      removeNonReplicableEdits(edit);
      HLogKey logKey = entry.getKey();
      // Don't replicate catalog entries, if the WALEdit wasn't
      // containing anything to replicate and if we're currently not set to replicate
      if (!(Bytes.equals(logKey.getTablename(), HConstants.ROOT_TABLE_NAME) ||
          Bytes.equals(logKey.getTablename(), HConstants.META_TABLE_NAME)) &&
          edit.size() != 0 && replicating.get()) {
        logKey.setClusterId(this.clusterId);
        currentNbOperations += countDistinctRowKeys(edit);
        currentNbEntries++;
      } else {
        this.metrics.logEditsFilteredRate.inc(1);
      }

View Full Code Here

    for(long i = 1; i < 101; i++) {
      if(i > 1 && i % 20 == 0) {
        hlog.rollWriter();
      }
      LOG.info(i);
      HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
          System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
      hlog.append(hri, test, edit, System.currentTimeMillis(), htd);
    }


    // Simulate a rapid insert that's followed

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.wal.HLogKey

com.google.protobuf.ByteString

com.ngdata.sep.impl.fork.ForkedReplicationSource

com.ngdata.sep.impl.SepConsumer

com.ngdata.sep.impl.SepReplicationSourceIT

org.apache.hadoop.hbase.mapreduce.TestWALPlayer

org.apache.hadoop.hbase.protobuf.ReplicationProtbufUtil

org.apache.hadoop.hbase.regionserver.HRegion

org.apache.hadoop.hbase.regionserver.TestHRegion

org.apache.hadoop.hbase.replication.regionserver.ReplicationSource

org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.