Package org.apache.hadoop.hbase.regionserver.wal

Examples of org.apache.hadoop.hbase.regionserver.wal.WALEdit


   * @param writeToWAL if true, then we should write to the log
   * @throws IOException
   */
  private void internalPut(Put put, UUID clusterId, boolean writeToWAL) throws IOException {
    Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
    WALEdit walEdit = new WALEdit();
    /* run pre put hook outside of lock to avoid deadlock */
    if (coprocessorHost != null) {
      if (coprocessorHost.prePut(put, walEdit, writeToWAL)) {
        return;
      }
View Full Code Here


                180000) / 2);
        long lastReport = EnvironmentEdgeManager.currentTimeMillis();

        while ((entry = reader.next()) != null) {
          HLogKey key = entry.getKey();
          WALEdit val = entry.getEdit();

          if (reporter != null) {
            intervalEdits += val.size();
            if (intervalEdits >= interval) {
              // Number of edits interval reached
              intervalEdits = 0;
              long cur = EnvironmentEdgeManager.currentTimeMillis();
              if (lastReport + period <= cur) {
                status.setStatus("Replaying edits..." +
                    " skipped=" + skippedEdits +
                    " edits=" + editsCount);
                // Timeout reached
                if(!reporter.progress()) {
                  msg = "Progressable reporter failed, stopping replay";
                  LOG.warn(msg);
                  status.abort(msg);
                  throw new IOException(msg);
                }
                reported_once = true;
                lastReport = cur;
              }
            }
          }

          // Start coprocessor replay here. The coprocessor is for each WALEdit
          // instead of a KeyValue.
          if (coprocessorHost != null) {
            status.setStatus("Running pre-WAL-restore hook in coprocessors");
            if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {
              // if bypass this log entry, ignore it ...
              continue;
            }
          }

          if (firstSeqIdInLog == -1) {
            firstSeqIdInLog = key.getLogSeqNum();
          }
          // Now, figure if we should skip this edit.
          if (key.getLogSeqNum() <= currentEditSeqId) {
            skippedEdits++;
            continue;
          }
          currentEditSeqId = key.getLogSeqNum();
          boolean flush = false;
          for (KeyValue kv: val.getKeyValues()) {
            // Check this edit is for me. Also, guard against writing the special
            // METACOLUMN info such as HBASE::CACHEFLUSH entries
            if (kv.matchingFamily(HLog.METAFAMILY) ||
                !Bytes.equals(key.getEncodedRegionName(), this.regionInfo.getEncodedNameAsBytes())) {
              skippedEdits++;
View Full Code Here

    try {
      // 1. run all pre-hooks before the atomic operation
      // if any pre hook indicates "bypass", bypass the entire operation

      // one WALEdit is used for all edits.
      WALEdit walEdit = new WALEdit();
      if (coprocessorHost != null) {
        for (Mutation m : mutations) {
          if (m instanceof Put) {
            if (coprocessorHost.prePut((Put) m, walEdit, m.getWriteToWAL())) {
              // by pass everything
              return;
            }
          } else if (m instanceof Delete) {
            Delete d = (Delete) m;
            prepareDelete(d);
            if (coprocessorHost.preDelete(d, walEdit, d.getWriteToWAL())) {
              // by pass everything
              return;
            }
          }
        }
      }

      long txid = 0;
      boolean walSyncSuccessful = false;
      boolean locked = false;

      // 2. acquire the row lock(s)
      acquiredLocks = new ArrayList<Integer>(rowsToLock.size());
      for (byte[] row : rowsToLock) {
        // attempt to lock all involved rows, fail if one lock times out
        Integer lid = getLock(null, row, true);
        if (lid == null) {
          throw new IOException("Failed to acquire lock on "
              + Bytes.toStringBinary(row));
        }
        acquiredLocks.add(lid);
      }

      // 3. acquire the region lock
      lock(this.updatesLock.readLock(), acquiredLocks.size());
      locked = true;

      // 4. Get a mvcc write number
      MultiVersionConsistencyControl.WriteEntry w = mvcc.beginMemstoreInsert();

      long now = EnvironmentEdgeManager.currentTimeMillis();
      byte[] byteNow = Bytes.toBytes(now);
      try {
        // 5. Check mutations and apply edits to a single WALEdit
        for (Mutation m : mutations) {
          if (m instanceof Put) {
            Map<byte[], List<KeyValue>> familyMap = m.getFamilyMap();
            checkFamilies(familyMap.keySet());
            checkTimestamps(familyMap, now);
            updateKVTimestamps(familyMap.values(), byteNow);
          } else if (m instanceof Delete) {
            Delete d = (Delete) m;
            prepareDelete(d);
            prepareDeleteTimestamps(d.getFamilyMap(), byteNow);
          } else {
            throw new DoNotRetryIOException(
                "Action must be Put or Delete. But was: "
                    + m.getClass().getName());
          }
          if (m.getWriteToWAL()) {
            addFamilyMapToWALEdit(m.getFamilyMap(), walEdit);
          }
        }

        // 6. append all edits at once (don't sync)
        if (walEdit.size() > 0) {
          txid = this.log.appendNoSync(regionInfo,
              this.htableDescriptor.getName(), walEdit,
              HConstants.DEFAULT_CLUSTER_ID, now, this.htableDescriptor);
        }

        // 7. apply to memstore
        long addedSize = 0;
        for (Mutation m : mutations) {
          addedSize += applyFamilyMapToMemstore(m.getFamilyMap(), w);
        }
        flush = isFlushSize(this.addAndGetGlobalMemstoreSize(addedSize));

        // 8. release region and row lock(s)
        this.updatesLock.readLock().unlock();
        locked = false;
        if (acquiredLocks != null) {
          for (Integer lid : acquiredLocks) {
            releaseRowLock(lid);
          }
          acquiredLocks = null;
        }

        // 9. sync WAL if required
        if (walEdit.size() > 0) {
          syncOrDefer(txid);
        }
        walSyncSuccessful = true;

        // 10. advance mvcc
View Full Code Here

      throws IOException {
    // TODO: Use MVCC to make this set of appends atomic to reads
    byte[] row = append.getRow();
    checkRow(row, "append");
    boolean flush = false;
    WALEdit walEdits = null;
    List<KeyValue> allKVs = new ArrayList<KeyValue>(append.size());
    Map<Store, List<KeyValue>> tempMemstore = new HashMap<Store, List<KeyValue>>();
    long before = EnvironmentEdgeManager.currentTimeMillis();
    long size = 0;
    long txid = 0;

    checkReadOnly();
    // Lock row
    startRegionOperation();
    this.writeRequestsCount.increment();
    try {
      Integer lid = getLock(lockid, row, true);
      lock(this.updatesLock.readLock());
      try {
        long now = EnvironmentEdgeManager.currentTimeMillis();
        // Process each family
        for (Map.Entry<byte[], List<KeyValue>> family : append.getFamilyMap()
            .entrySet()) {

          Store store = stores.get(family.getKey());
          List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());

          // Get previous values for all columns in this family
          Get get = new Get(row);
          for (KeyValue kv : family.getValue()) {
            get.addColumn(family.getKey(), kv.getQualifier());
          }
          List<KeyValue> results = get(get, false);

          // Iterate the input columns and update existing values if they were
          // found, otherwise add new column initialized to the append value

          // Avoid as much copying as possible. Every byte is copied at most
          // once.
          // Would be nice if KeyValue had scatter/gather logic
          int idx = 0;
          for (KeyValue kv : family.getValue()) {
            KeyValue newKV;
            if (idx < results.size()
                && results.get(idx).matchingQualifier(kv.getBuffer(),
                    kv.getQualifierOffset(), kv.getQualifierLength())) {
              KeyValue oldKv = results.get(idx);
              // allocate an empty kv once
              newKV = new KeyValue(row.length, kv.getFamilyLength(),
                  kv.getQualifierLength(), now, KeyValue.Type.Put,
                  oldKv.getValueLength() + kv.getValueLength());
              // copy in the value
              System.arraycopy(oldKv.getBuffer(), oldKv.getValueOffset(),
                  newKV.getBuffer(), newKV.getValueOffset(),
                  oldKv.getValueLength());
              System.arraycopy(kv.getBuffer(), kv.getValueOffset(),
                  newKV.getBuffer(),
                  newKV.getValueOffset() + oldKv.getValueLength(),
                  kv.getValueLength());
              idx++;
            } else {
              // allocate an empty kv once
              newKV = new KeyValue(row.length, kv.getFamilyLength(),
                  kv.getQualifierLength(), now, KeyValue.Type.Put,
                  kv.getValueLength());
              // copy in the value
              System.arraycopy(kv.getBuffer(), kv.getValueOffset(),
                  newKV.getBuffer(), newKV.getValueOffset(),
                  kv.getValueLength());
            }
            // copy in row, family, and qualifier
            System.arraycopy(kv.getBuffer(), kv.getRowOffset(),
                newKV.getBuffer(), newKV.getRowOffset(), kv.getRowLength());
            System.arraycopy(kv.getBuffer(), kv.getFamilyOffset(),
                newKV.getBuffer(), newKV.getFamilyOffset(),
                kv.getFamilyLength());
            System.arraycopy(kv.getBuffer(), kv.getQualifierOffset(),
                newKV.getBuffer(), newKV.getQualifierOffset(),
                kv.getQualifierLength());

            kvs.add(newKV);

            // Append update to WAL
            if (writeToWAL) {
              if (walEdits == null) {
                walEdits = new WALEdit();
              }
              walEdits.add(newKV);
            }
          }

          // store the kvs to the temporary memstore before writing HLog
          tempMemstore.put(store, kvs);
View Full Code Here

    // TODO: Use MVCC to make this set of increments atomic to reads
    byte [] row = increment.getRow();
    checkRow(row, "increment");
    TimeRange tr = increment.getTimeRange();
    boolean flush = false;
    WALEdit walEdits = null;
    List<KeyValue> allKVs = new ArrayList<KeyValue>(increment.numColumns());
    Map<Store, List<KeyValue>> tempMemstore = new HashMap<Store, List<KeyValue>>();
    long before = EnvironmentEdgeManager.currentTimeMillis();
    long size = 0;
    long txid = 0;

    checkReadOnly();
    // Lock row
    startRegionOperation();
    this.writeRequestsCount.increment();
    try {
      Integer lid = getLock(lockid, row, true);
      lock(this.updatesLock.readLock());
      try {
        long now = EnvironmentEdgeManager.currentTimeMillis();
        // Process each family
        for (Map.Entry<byte [], NavigableMap<byte [], Long>> family :
          increment.getFamilyMap().entrySet()) {

          Store store = stores.get(family.getKey());
          List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());

          // Get previous values for all columns in this family
          Get get = new Get(row);
          for (Map.Entry<byte [], Long> column : family.getValue().entrySet()) {
            get.addColumn(family.getKey(), column.getKey());
          }
          get.setTimeRange(tr.getMin(), tr.getMax());
          List<KeyValue> results = get(get, false);

          // Iterate the input columns and update existing values if they were
          // found, otherwise add new column initialized to the increment amount
          int idx = 0;
          for (Map.Entry<byte [], Long> column : family.getValue().entrySet()) {
            long amount = column.getValue();
            if (idx < results.size() &&
                results.get(idx).matchingQualifier(column.getKey())) {
              KeyValue kv = results.get(idx);
              if(kv.getValueLength() == Bytes.SIZEOF_LONG) {
                amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), Bytes.SIZEOF_LONG);
              } else {
                // throw DoNotRetryIOException instead of IllegalArgumentException
                throw new DoNotRetryIOException(
                    "Attempted to increment field that isn't 64 bits wide");
              }
              idx++;
            }

            // Append new incremented KeyValue to list
            KeyValue newKV = new KeyValue(row, family.getKey(), column.getKey(),
                now, Bytes.toBytes(amount));
            kvs.add(newKV);

            // Append update to WAL
            if (writeToWAL) {
              if (walEdits == null) {
                walEdits = new WALEdit();
              }
              walEdits.add(newKV);
            }
          }

          //store the kvs to the temporary memstore before writing HLog
          tempMemstore.put(store, kvs);
View Full Code Here

            Bytes.toBytes(result));

          // now log it:
          if (writeToWAL) {
            long now = EnvironmentEdgeManager.currentTimeMillis();
            WALEdit walEdit = new WALEdit();
            walEdit.add(newKv);
            // Using default cluster id, as this can only happen in the
            // orginating cluster. A slave cluster receives the final value (not
            // the delta) as a Put.
            txid = this.log.appendNoSync(regionInfo, this.htableDescriptor.getName(),
                walEdit, HConstants.DEFAULT_CLUSTER_ID, now,
View Full Code Here

    // coprocessor.
    // Use a Put to create familyMap.
    Put p = creatPutWith2Families(TEST_ROW);

    Map<byte [], List<KeyValue>> familyMap = p.getFamilyMap();
    WALEdit edit = new WALEdit();
    addFamilyMapToWALEdit(familyMap, edit);

    boolean foundFamily0 = false;
    boolean foundFamily2 = false;
    boolean modifiedFamily1 = false;

    List<KeyValue> kvs = edit.getKeyValues();

    for (KeyValue kv : kvs) {
      if (Arrays.equals(kv.getFamily(), TEST_FAMILY[0])) {
        foundFamily0 = true;
      }
View Full Code Here


    //HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf);
    HLog wal = createWAL(this.conf);
    //Put p = creatPutWith2Families(TEST_ROW);
    WALEdit edit = new WALEdit();
    long now = EnvironmentEdgeManager.currentTimeMillis();
    //addFamilyMapToWALEdit(p.getFamilyMap(), edit);
    final int countPerFamily = 1000;
    //for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
    for (HColumnDescriptor hcd: htd.getFamilies()) {
View Full Code Here

  throws IOException {
    String familyStr = Bytes.toString(family);
    for (int j = 0; j < count; j++) {
      byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
      byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
      WALEdit edit = new WALEdit();
      edit.add(new KeyValue(rowName, family, qualifierBytes,
        ee.currentTimeMillis(), columnBytes));
      wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
    }
  }
View Full Code Here

   * @throws IOException
   */
  private void internalDelete(Delete delete, UUID clusterId,
      boolean writeToWAL) throws IOException {
    Map<byte[], List<KeyValue>> familyMap = delete.getFamilyMap();
    WALEdit walEdit = new WALEdit();
    /* Run coprocessor pre hook outside of locks to avoid deadlock */
    if (coprocessorHost != null) {
      if (coprocessorHost.preDelete(delete, walEdit, writeToWAL)) {
        return;
      }
    }

    long now = EnvironmentEdgeManager.currentTimeMillis();
    byte [] byteNow = Bytes.toBytes(now);
    boolean flush = false;

    lock(updatesLock.readLock());
    try {
      prepareDeleteTimestamps(delete.getFamilyMap(), byteNow);

      if (writeToWAL) {
        // write/sync to WAL should happen before we touch memstore.
        //
        // If order is reversed, i.e. we write to memstore first, and
        // for some reason fail to write/sync to commit log, the memstore
        // will contain uncommitted transactions.
        //
        // bunch up all edits across all column families into a
        // single WALEdit.
        addFamilyMapToWALEdit(familyMap, walEdit);
        walEdit.addClusterIds(delete.getClusterIds());
        this.log.append(regionInfo, this.htableDescriptor.getName(),
            walEdit, clusterId, now, this.htableDescriptor);
      }

      // Now make changes to the memstore.
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.wal.WALEdit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.