Thread chiThread = new Thread(new Runnable() {
public void run() {
try {
setStatusString("Preprocessing: reading data from HDFS: " + location);
final FastSharder sharder = createSharder(getGraphName(), getNumShards());
HDFSGraphLoader hdfsLoader = new HDFSGraphLoader(location, new EdgeProcessor<Float>() {
long counter = 0;
public Float receiveEdge(int from, int to, String token) {
try {
sharder.addEdge(from, to, token);
counter++;
if (counter % 100000 == 0) {
setStatusString("Preprocessing, read " + counter + " edges");
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
});
hdfsLoader.load(pigSplit.getConf());
setStatusString("Sharding...");
sharder.process();
logger.info("Starting to run GraphChi");
setStatusString("Start GraphChi engine");
runGraphChi();
logger.info("Ready.");