Package org.apache.hadoop.hbase.util

Examples of org.apache.hadoop.hbase.util.ManualEnvironmentEdge


    // TODO: This test would seem to presume hardcoded RETRY_BACKOFF which it should not.
    final long ANY_PAUSE = 100;
    ServerName location = ServerName.valueOf("127.0.0.1", 1, 0);
    ServerName diffLocation = ServerName.valueOf("127.0.0.1", 2, 0);

    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
    EnvironmentEdgeManager.injectEdge(timeMachine);
    try {
      long timeBase = timeMachine.currentTime();
      long largeAmountOfTime = ANY_PAUSE * 1000;
      ConnectionManager.ServerErrorTracker tracker =
          new ConnectionManager.ServerErrorTracker(largeAmountOfTime, 100);

      // The default backoff is 0.
      assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // Check some backoff values from HConstants sequence.
      tracker.reportServerError(location);
      assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(location, ANY_PAUSE));
      tracker.reportServerError(location);
      tracker.reportServerError(location);
      tracker.reportServerError(location);
      assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // All of this shouldn't affect backoff for different location.
      assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
      tracker.reportServerError(diffLocation);
      assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));

      // Check with different base.
      assertEqualsWithJitter(ANY_PAUSE * 10,
          tracker.calculateBackoffTime(location, ANY_PAUSE * 2));

      // See that time from last error is taken into account. Time shift is applied after jitter,
      // so pass the original expected backoff as the base for jitter.
      long timeShift = (long)(ANY_PAUSE * 0.5);
      timeMachine.setValue(timeBase + timeShift);
      assertEqualsWithJitter((ANY_PAUSE * 5) - timeShift,
        tracker.calculateBackoffTime(location, ANY_PAUSE), ANY_PAUSE * 2);

      // However we should not go into negative.
      timeMachine.setValue(timeBase + ANY_PAUSE * 100);
      assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // We also should not go over the boundary; last retry would be on it.
      long timeLeft = (long)(ANY_PAUSE * 0.5);
      timeMachine.setValue(timeBase + largeAmountOfTime - timeLeft);
      assertTrue(tracker.canRetryMore(1));
      tracker.reportServerError(location);
      assertEquals(timeLeft, tracker.calculateBackoffTime(location, ANY_PAUSE));
      timeMachine.setValue(timeBase + largeAmountOfTime);
      assertFalse(tracker.canRetryMore(1));
    } finally {
      EnvironmentEdgeManager.reset();
    }
  }
View Full Code Here


@Category(SmallTests.class)
public class TestFlushRegionEntry {
  @Before
  public void setUp() throws Exception {
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    edge.setValue(12345);
    EnvironmentEdgeManager.injectEdge(edge);
  }
View Full Code Here

  @Test
  public void testICV_negMemstoreSize()  throws IOException {
      init(this.name.getMethodName());

    long time = 100;
    ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
    ee.setValue(time);
    EnvironmentEdgeManagerTestHelper.injectEdge(ee);
    long newValue = 3L;
    long size = 0;


    size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue)));
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue)));
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue)));
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue)));
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue)));


    for ( int i = 0 ; i < 10000 ; ++i) {
      newValue++;

      long ret = this.store.updateColumnValue(row, family, qf1, newValue);
      long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);

      if (ret != 0) System.out.println("ret: " + ret);
      if (ret2 != 0) System.out.println("ret2: " + ret2);

      Assert.assertTrue("ret: " + ret, ret >= 0);
      size += ret;
      Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
      size += ret2;


      if (i % 1000 == 0)
        ee.setValue(++time);
    }

    long computedSize=0;
    for (KeyValue kv : this.store.memstore.kvset) {
      long kvsize = MemStore.heapSizeChange(kv, true);
View Full Code Here

    Assert.assertEquals(computedSize, size);
  }

  @Test
  public void testIncrementColumnValue_SnapshotFlushCombo() throws Exception {
    ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
    EnvironmentEdgeManagerTestHelper.injectEdge(mee);
    init(this.name.getMethodName());

    long oldValue = 1L;
    long newValue = 3L;
    this.store.add(new KeyValue(row, family, qf1,
        EnvironmentEdgeManager.currentTimeMillis(),
        Bytes.toBytes(oldValue)));

    // snapshot the store.
    this.store.snapshot();

    // update during the snapshot, the exact same TS as the Put (lololol)
    long ret = this.store.updateColumnValue(row, family, qf1, newValue);

    // memstore should have grown by some amount.
    Assert.assertTrue(ret > 0);

    // then flush.
    flushStore(store, id++);
    Assert.assertEquals(1, this.store.getStorefiles().size());
    Assert.assertEquals(1, this.store.memstore.kvset.size());

    // now increment again:
    newValue += 1;
    this.store.updateColumnValue(row, family, qf1, newValue);

    // at this point we have a TS=1 in snapshot, and a TS=2 in kvset, so increment again:
    newValue += 1;
    this.store.updateColumnValue(row, family, qf1, newValue);

    // the second TS should be TS=2 or higher., even though 'time=1' right now.


    // how many key/values for this row are there?
    Get get = new Get(row);
    get.addColumn(family, qf1);
    get.setMaxVersions(); // all versions.
    List<Cell> results = new ArrayList<Cell>();

    results = HBaseTestingUtility.getFromStoreFile(store, get);
    Assert.assertEquals(2, results.size());

    long ts1 = results.get(0).getTimestamp();
    long ts2 = results.get(1).getTimestamp();

    Assert.assertTrue(ts1 > ts2);
    Assert.assertEquals(newValue, Bytes.toLong(CellUtil.cloneValue(results.get(0))));
    Assert.assertEquals(oldValue, Bytes.toLong(CellUtil.cloneValue(results.get(1))));

    mee.setValue(2); // time goes up slightly
    newValue += 1;
    this.store.updateColumnValue(row, family, qf1, newValue);

    results = HBaseTestingUtility.getFromStoreFile(store, get);
    Assert.assertEquals(2, results.size());
View Full Code Here

@Category({RegionServerTests.class, SmallTests.class})
public class TestFlushRegionEntry {
  @Before
  public void setUp() throws Exception {
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    edge.setValue(12345);
    EnvironmentEdgeManager.injectEdge(edge);
  }
View Full Code Here

    Put put = new Put(ROW);
    put.add(FAM_NAM, ROW, ROW);
    table.put(put);

    ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
    mee.setValue(System.currentTimeMillis());
    EnvironmentEdgeManager.injectEdge(mee);
    LOG.info("first get");
    table.get(new Get(ROW));

    LOG.info("first get - changing the time & sleeping");
    mee.incValue(idleTime + 1000);
    Thread.sleep(1500); // we need to wait a little for the connection to be seen as idle.
                        // 1500 = sleep time in RpcClient#waitForWork + a margin

    LOG.info("second get - connection has been marked idle in the middle");
    // To check that the connection actually became idle would need to read some private
    //  fields of RpcClient.
    table.get(new Get(ROW));
    mee.incValue(idleTime + 1000);

    LOG.info("third get - connection is idle, but the reader doesn't know yet");
    // We're testing here a special case:
    //  time limit reached BUT connection not yet reclaimed AND a new call.
    //  in this situation, we don't close the connection, instead we use it immediately.
View Full Code Here

    // TODO: This test would seem to presume hardcoded RETRY_BACKOFF which it should not.
    final long ANY_PAUSE = 100;
    ServerName location = ServerName.valueOf("127.0.0.1", 1, 0);
    ServerName diffLocation = ServerName.valueOf("127.0.0.1", 2, 0);

    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
    EnvironmentEdgeManager.injectEdge(timeMachine);
    try {
      long timeBase = timeMachine.currentTime();
      long largeAmountOfTime = ANY_PAUSE * 1000;
      ConnectionManager.ServerErrorTracker tracker =
          new ConnectionManager.ServerErrorTracker(largeAmountOfTime, 100);

      // The default backoff is 0.
      assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // Check some backoff values from HConstants sequence.
      tracker.reportServerError(location);
      assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(location, ANY_PAUSE));
      tracker.reportServerError(location);
      tracker.reportServerError(location);
      tracker.reportServerError(location);
      assertEqualsWithJitter(ANY_PAUSE * 5, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // All of this shouldn't affect backoff for different location.
      assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));
      tracker.reportServerError(diffLocation);
      assertEqualsWithJitter(ANY_PAUSE, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE));

      // Check with different base.
      assertEqualsWithJitter(ANY_PAUSE * 10,
          tracker.calculateBackoffTime(location, ANY_PAUSE * 2));

      // See that time from last error is taken into account. Time shift is applied after jitter,
      // so pass the original expected backoff as the base for jitter.
      long timeShift = (long)(ANY_PAUSE * 0.5);
      timeMachine.setValue(timeBase + timeShift);
      assertEqualsWithJitter((ANY_PAUSE * 5) - timeShift,
        tracker.calculateBackoffTime(location, ANY_PAUSE), ANY_PAUSE * 2);

      // However we should not go into negative.
      timeMachine.setValue(timeBase + ANY_PAUSE * 100);
      assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE));

      // We also should not go over the boundary; last retry would be on it.
      long timeLeft = (long)(ANY_PAUSE * 0.5);
      timeMachine.setValue(timeBase + largeAmountOfTime - timeLeft);
      assertTrue(tracker.canRetryMore(1));
      tracker.reportServerError(location);
      assertEquals(timeLeft, tracker.calculateBackoffTime(location, ANY_PAUSE));
      timeMachine.setValue(timeBase + largeAmountOfTime);
      assertFalse(tracker.canRetryMore(1));
    } finally {
      EnvironmentEdgeManager.reset();
    }
  }
View Full Code Here

    } catch (AssertionError err) {}
  }

  @Test
  public void testCleanup() throws Exception {
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    EnvironmentEdgeManager.injectEdge(edge);
    try {
      ServerNonceManager nm = createManager(6);
      Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class));
      edge.setValue(1);
      assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
      assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
      assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
      edge.setValue(2);
      nm.endOperation(NO_NONCE, 1, true);
      edge.setValue(4);
      nm.endOperation(NO_NONCE, 2, true);
      edge.setValue(9);
      cleanup.choreForTesting();
      // Nonce 1 has been cleaned up.
      assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
      // Nonce 2 has not been cleaned up.
      assertFalse(nm.startOperation(NO_NONCE, 2, createStoppable()));
      // Nonce 3 was active and active ops should never be cleaned up; try to end and start.
      nm.endOperation(NO_NONCE, 3, false);
      assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
      edge.setValue(11);
      cleanup.choreForTesting();
      // Now, nonce 2 has been cleaned up.
      assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
    } finally {
      EnvironmentEdgeManager.reset();
View Full Code Here

    }
  }

  @Test
  public void testWalNonces() throws Exception {
    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    EnvironmentEdgeManager.injectEdge(edge);
    try {
      ServerNonceManager nm = createManager(6);
      Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class));
      // Add nonces from WAL, including dups.
      edge.setValue(12);
      nm.reportOperationFromWal(NO_NONCE, 1, 8);
      nm.reportOperationFromWal(NO_NONCE, 2, 2);
      nm.reportOperationFromWal(NO_NONCE, 3, 5);
      nm.reportOperationFromWal(NO_NONCE, 3, 6);
      // WAL nonces should prevent cross-server conflicts.
      assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable()));
      // Make sure we ignore very old nonces, but not borderline old nonces.
      assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
      assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable()));
      // Make sure grace period is counted from recovery time.
      edge.setValue(17);
      cleanup.choreForTesting();
      assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable()));
      assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable()));
      edge.setValue(19);
      cleanup.choreForTesting();
      assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
      assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
    } finally {
      EnvironmentEdgeManager.reset();
View Full Code Here

  @Test
  public void testICV_negMemstoreSize()  throws IOException {
      init(this.name.getMethodName());

    long time = 100;
    ManualEnvironmentEdge ee = new ManualEnvironmentEdge();
    ee.setValue(time);
    EnvironmentEdgeManagerTestHelper.injectEdge(ee);
    long newValue = 3L;
    long size = 0;


    size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue))).getFirst();
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue))).getFirst();
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue))).getFirst();
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue))).getFirst();
    size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
        System.currentTimeMillis(),
        Bytes.toBytes(newValue))).getFirst();


    for ( int i = 0 ; i < 10000 ; ++i) {
      newValue++;

      long ret = this.store.updateColumnValue(row, family, qf1, newValue);
      long ret2 = this.store.updateColumnValue(row2, family, qf1, newValue);

      if (ret != 0) System.out.println("ret: " + ret);
      if (ret2 != 0) System.out.println("ret2: " + ret2);

      Assert.assertTrue("ret: " + ret, ret >= 0);
      size += ret;
      Assert.assertTrue("ret2: " + ret2, ret2 >= 0);
      size += ret2;


      if (i % 1000 == 0)
        ee.setValue(++time);
    }

    long computedSize=0;
    for (Cell cell : ((DefaultMemStore)this.store.memstore).cellSet) {
      long kvsize = DefaultMemStore.heapSizeChange(cell, true);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hbase.util.ManualEnvironmentEdge

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.