// Copyright 2006 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.enterprise.connector.scheduler;
import com.google.enterprise.connector.pusher.MockFeedConnection;
import com.google.enterprise.connector.traversal.BatchResult;
import com.google.enterprise.connector.traversal.BatchSize;
import com.google.enterprise.connector.traversal.FileSizeLimitInfo;
import com.google.enterprise.connector.traversal.TraversalDelayPolicy;
import com.google.enterprise.connector.util.testing.AdjustableClock;
import junit.framework.TestCase;
/**
* Test HostLoadManager class.
*/
public class HostLoadManagerTest extends TestCase {
static AdjustableClock clock = new AdjustableClock();
// Adjust the current time to the minimum milliseconds of a second.
// This can help us avoid running accross 1-second boundaries and
// splitting results between two periods.
private void alignTime(int min) {
long now = clock.getTimeMillis();
clock.setTimeMillis(now - (now % 1000L) + min);
}
private HostLoadManager newHostLoadManager(int load) {
return newHostLoadManager(load, 200);
}
private HostLoadManager newHostLoadManager(int load, int batchsize) {
alignTime(50);
HostLoadManager hlm = new HostLoadManager(null, null, clock);
hlm.setLoad(load);
hlm.setBatchSize(batchsize);
return hlm;
}
private BatchResult newBatchResult(int numDocs) {
return newBatchResult(numDocs, 250);
}
private BatchResult newBatchResult(int numDocs, int duration) {
long now = clock.getTimeMillis();
return new BatchResult(TraversalDelayPolicy.IMMEDIATE, numDocs,
now - duration, now);
}
public void testMaxFeedRateLimit() {
HostLoadManager hostLoadManager = newHostLoadManager(60);
assertEquals(60, hostLoadManager.determineBatchSize().getHint());
hostLoadManager.recordResult(newBatchResult(60));
assertEquals(0, hostLoadManager.determineBatchSize().getHint());
}
public void testPeriod() {
HostLoadManager hostLoadManager = newHostLoadManager(60);
hostLoadManager.setPeriod(1); // 1 second.
hostLoadManager.recordResult(newBatchResult(60));
assertEquals(0, hostLoadManager.determineBatchSize().getHint());
// Advance time more than one second so that delay expires.
clock.adjustTime(1250); // 1.25 second
assertTrue(hostLoadManager.determineBatchSize().getHint() > 0);
}
/**
* Test that if we meet or exceed the host load limit, further
* traversal should be delayed until the next traversal period.
*/
public void testShouldDelay() {
HostLoadManager hostLoadManager = newHostLoadManager(60);
hostLoadManager.setPeriod(1); // 1 second.
assertFalse(hostLoadManager.shouldDelay());
hostLoadManager.recordResult(newBatchResult(60));
assertTrue(hostLoadManager.shouldDelay());
// Advance time more than 1 second, the LoadManager period, so that
// this connector should be allowed to run again without delay.
clock.adjustTime(1250); // 1.25 second
assertFalse(hostLoadManager.shouldDelay());
}
/**
* Test minimum batchSize.
*/
public void testMinimumBatchSize() {
HostLoadManager hostLoadManager = newHostLoadManager(60);
hostLoadManager.setBatchSize(10);
BatchSize batchSize = hostLoadManager.determineBatchSize();
assertEquals(10, batchSize.getHint());
hostLoadManager.setBatchSize(20);
batchSize = hostLoadManager.determineBatchSize();
assertEquals(20, batchSize.getHint());
hostLoadManager.setBatchSize(100);
batchSize = hostLoadManager.determineBatchSize();
assertEquals(60, batchSize.getHint());
}
/**
* Test that tiny batch sizes are not returned. If nearly full batches
* are returned fast enought, the algorithm could trend toward 0, so
* the load manager sets a floor.
*/
public void testNoTinyBatch() {
HostLoadManager hostLoadManager = newHostLoadManager(200);
hostLoadManager.setPeriod(1); // 1 second.
assertFalse(hostLoadManager.shouldDelay());
hostLoadManager.recordResult(newBatchResult(199, 10));
// Advance time more than 1 second, the LoadManager period, so that
// this connector should be allowed to run again without delay.
clock.adjustTime(1250); // 1.25 second
// At that throughput, a batch size of 1 or 2 would be calculated.
BatchSize batchSize = hostLoadManager.determineBatchSize();
assertEquals(200, hostLoadManager.determineBatchSize().getHint());
assertFalse(hostLoadManager.shouldDelay());
}
/**
* Test that if we returned a huge result that exceeds the load,
* we will delay further traversals accordingly.
* Regression for Issue 194.
*/
public void testShouldDelayAfterHugeBatch() {
HostLoadManager hostLoadManager = newHostLoadManager(50);
hostLoadManager.setPeriod(1); // 1 second.
assertFalse(hostLoadManager.shouldDelay());
hostLoadManager.recordResult(newBatchResult(150));
// We should delay, after grossly exceeding the load.
assertTrue(hostLoadManager.shouldDelay());
// Advance time more than 1 second, the LoadManager period.
// We should still delay, paying the penalty for grossly exceeding
// the load previously.
clock.adjustTime(1250); // 1.25 second
assertTrue(hostLoadManager.shouldDelay());
// Advance another couple of seconds, allowing the penalty to expire.
// This will bring the average load over the elapsed traversal time
// plus the penalty periods in line with the configured load.
clock.adjustTime(2500); // 2.5 seconds
// We should then be allowed to traverse again.
assertFalse(hostLoadManager.shouldDelay());
}
/**
* The new determineBatchSize logic treats both the hint and the load
* as "suggestions", or "desired targets". The traversal might
* return more than the batchHint, and the number of docs processed in
* a period might exceed the load. This flexibility allows the connectors
* to work more efficiently without expending a great deal of effort
* trying to hit the batch size exactly.
*/
public void testDetermineBatchSize() {
HostLoadManager hostLoadManager = newHostLoadManager(60);
BatchSize batchSize = hostLoadManager.determineBatchSize();
assertEquals(60, batchSize.getHint());
hostLoadManager.setBatchSize(40);
batchSize = hostLoadManager.determineBatchSize();
assertEquals(40, batchSize.getHint());
}
/**
* Test that the throughput is spread out for long running batches.
* For instance, if the period is 1 minute, and the load is 200;
* then a traversal that took 4 minutes, but returned 200 documents
* only averaged 50 docs per period. Regression for Issue 189.
*/
public void testAmortizeLongRunningBatch() {
HostLoadManager hostLoadManager = newHostLoadManager(200, 1000);
hostLoadManager.setPeriod(1); // 1 second
// We don't want to register the result for this test too close
// to the beginning or end of a second.
alignTime(450);
// If we are staying well below the load limit per period,
// we should not delay.
assertFalse(hostLoadManager.shouldDelay());
hostLoadManager.recordResult(newBatchResult(200, 4000));
assertFalse(hostLoadManager.shouldDelay());
// The host load manager will try to suggest larger batches,
// trying to pull the low throughput up towards the load.
BatchSize batchSize = hostLoadManager.determineBatchSize();
assertEquals(800, batchSize.getHint());
// Another slow-running batch should produce a similar result, but
// the returned batch hint should be capped at the maximum batch size.
hostLoadManager.recordResult(newBatchResult(800, 8000));
assertFalse(hostLoadManager.shouldDelay());
batchSize = hostLoadManager.determineBatchSize();
assertEquals(1000, batchSize.getHint());
// A gross overage should put the brakes on.
hostLoadManager.recordResult(newBatchResult(4000, 200));
assertTrue(hostLoadManager.shouldDelay());
batchSize = hostLoadManager.determineBatchSize();
assertEquals(0, batchSize.getHint());
}
/**
* Test shouldDelay(void) with a low memory condition.
*/
/* TODO (bmj): This test fails too frequently depending upon the environment.
* Low memory detection and handling should be completely re-evaluated.
* Disabling this test for now.
*/
/*
public void testShouldDelayLowMemory() {
Runtime rt = Runtime.getRuntime();
FileSizeLimitInfo fsli = new FileSizeLimitInfo();
HostLoadManager hostLoadManager = new HostLoadManager(null, fsli, clock);
// OK to start a traversal if there is plenty of memory for a new feed.
rt.gc();
fsli.setMaxFeedSize(rt.freeMemory()/100);
assertFalse(hostLoadManager.shouldDelay());
// Not OK to start a traversal if there is not enough memory for feeds.
rt.gc();
fsli.setMaxFeedSize(rt.maxMemory() - (rt.totalMemory() - rt.freeMemory()));
assertTrue(hostLoadManager.shouldDelay());
// If nothing changes, it should still delay.
clock.adjustTime(250); // .25 second
assertTrue(hostLoadManager.shouldDelay());
// Clearing the low memory condition should make it better.
rt.gc();
fsli.setMaxFeedSize(rt.freeMemory()/100);
assertFalse(hostLoadManager.shouldDelay());
}
*/
/**
* Test shouldDelay(void) with backlogged FeedConnection.
*/
public void testShouldDelayFeedBacklogged() {
BacklogFeedConnection feedConnection = new BacklogFeedConnection();
HostLoadManager hostLoadManager = new HostLoadManager(feedConnection, null, clock);
// OK to start a traversal if feedConnection is not backlogged.
feedConnection.setBacklogged(false);
assertFalse(hostLoadManager.shouldDelay());
// Not OK to start a traversal if feedConnection is backlogged.
feedConnection.setBacklogged(true);
assertTrue(hostLoadManager.shouldDelay());
}
/**
* A FeedConnection that can be backlogged.
*/
private class BacklogFeedConnection extends MockFeedConnection {
private boolean backlogged = false;
public void setBacklogged(boolean backlogged) {
this.backlogged = backlogged;
}
@Override
public boolean isBacklogged() {
return backlogged;
}
}
}