package org.archive.wayback.accesscontrol.robotstxt.redis;
import java.io.FileInputStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.archive.wayback.accesscontrol.robotstxt.redis.RedisRobotsLogic.KeyRedisValue;
import org.archive.wayback.exception.LiveWebCacheUnavailableException;
import org.archive.wayback.liveweb.LiveRobotsNoCache;
import org.archive.wayback.webapp.PerfStats;
public class RedisRefresher extends SimpleRedisRobotsCache {
private final static Logger LOGGER = Logger.getLogger(RedisRefresher.class.getName());
protected ExecutorService refreshService;
protected Set<String> activeUrls;
public void setRefreshService(ExecutorService refresh)
{
refreshService = refresh;
}
public static void main(String[] args)
{
if (args.length < 1) {
System.err.println("USAGE: <properties file>");
System.exit(-1);
}
String propsFile = args[0];
Properties props = new Properties();
try {
props.load(new FileInputStream(propsFile));
} catch (Exception e) {
e.printStackTrace();
}
// Init Live Web
LiveRobotsNoCache liveweb = new LiveRobotsNoCache();
liveweb.setMaxRobotsSize(Integer.parseInt(props.getProperty("liveweb.maxRobotsSize", "512000")));
liveweb.setMaxHostConnections(Integer.parseInt(props.getProperty("liveweb.maxHostConn", "500")));
int maxTotalConn = Integer.parseInt(props.getProperty("liveweb.maxTotalConn", "500"));
liveweb.setMaxTotalConnections(maxTotalConn);
//liveweb.setProxyHostPort(props.getProperty("liveweb.proxyHostPort"));
liveweb.setConnectionTimeoutMS(Integer.parseInt(props.getProperty("liveweb.timeout", "10000")));
liveweb.setConnectionTimeoutMS(Integer.parseInt(props.getProperty("liveweb.timeout", "10000")));
// Init Redis Conn
RedisConnectionManager redisConnMan = new RedisConnectionManager();
redisConnMan.setHost(props.getProperty("redis.host"));
redisConnMan.setPort(Integer.parseInt(props.getProperty("redis.port")));
redisConnMan.setPassword(props.getProperty("redis.password"));
redisConnMan.setTimeout(Integer.parseInt(props.getProperty("redis.timeout")));
redisConnMan.setConnections(Integer.parseInt(props.getProperty("redis.maxConnections")));
redisConnMan.init();
ExecutorService refreshService = Executors.newFixedThreadPool(maxTotalConn);
RedisRefresher refresher = new RedisRefresher();
refresher.setLiveweb(liveweb);
refresher.setRedisConnMan(redisConnMan);
refresher.setRefreshService(refreshService);
refresher.setGzipRobots(Boolean.parseBoolean(props.getProperty("redis.gzipRobots", "false")));
Logger.getLogger(PerfStats.class.getName()).setLevel(Level.OFF);
refresher.processRedisUpdateQueue();
}
protected void processRedisUpdateQueue()
{
int errorCounter = 0;
int maxErrorThresh = 10;
int errorSleepTime = 10000;
int maxQueued = 500;
int currQSize = 0;
activeUrls = new HashSet<String>();
activeUrls = Collections.synchronizedSet(activeUrls);
try {
while (true) {
if (errorCounter >= maxErrorThresh) {
LOGGER.warning(errorCounter + " Redis ERRORS! Sleeping for " + errorSleepTime);
Thread.sleep(errorSleepTime);
}
currQSize = activeUrls.size();
if (currQSize >= maxQueued) {
Thread.sleep(100);
continue;
} else {
Thread.sleep(0);
}
KeyRedisValue value = null;
//long startTime = System.currentTimeMillis();
try {
value = redisCmds.popKeyAndGet(UPDATE_QUEUE_KEY);
errorCounter = 0;
} catch (LiveWebCacheUnavailableException e) {
errorCounter++;
} catch (Exception exc) {
errorCounter = maxErrorThresh;
LOGGER.log(Level.SEVERE, "REDIS SEVERE", exc);
} finally {
//PerformanceLogger.noteElapsed("PopKeyAndGet", System.currentTimeMillis() - startTime);
}
if (value == null) {
continue;
}
String url = value.key;
if (!isExpired(value, url, 0)) {
continue;
}
if (activeUrls.contains(url)) {
continue;
}
refreshService.execute(new ForceUpdater(url));
}
} catch (InterruptedException e) {
LOGGER.info("Interrupted, Quitting");
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "UPDATER SEVERE", e);
} finally {
shutdown();
}
}
private class ForceUpdater implements Runnable
{
private String url;
private ForceUpdater(String url)
{
this.url = url;
}
@Override
public void run()
{
RobotsResult result = forceUpdate(url, 0, true);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info((!result.isSameRobots() ? "UPDATE " : "NOCHANGE ") + url);
}
if (activeUrls != null) {
activeUrls.remove(url);
}
}
}
}