package benchmarks.idf;
import benchmarks.Evaluator;
import edu.umass.pql.container.PDefaultMap;
import edu.umass.pql.Env;
import java.util.*;
import benchmarks.webgraph.Webdoc;
import benchmarks.webgraph.Link;
import benchmarks.webgraph.Generator;
import java.util.concurrent.*;
public class ParaManual extends Evaluator.ParaEvaluator
{
public static final boolean SIMPLE_MERGE = false;
static int MERGE_THREAD_LIMIT = getMergeThreadLimit();
//static ExecutorService thread_pool = Executors.newFixedThreadPool(Env.THREADS_NR);
static CyclicBarrier[] barriers = init_barriers(Env.THREADS_NR);
static CyclicBarrier[]
init_barriers(int nr)
{
final CyclicBarrier[] retval = new CyclicBarrier[nr];
for (int i = 0; i < nr; i++)
retval[i] = new CyclicBarrier(2);
return retval;
}
public static int
getMergeThreadLimit()
{
Map<String, String> env = System.getenv();
final String limit_str = env.get("PQL_MERGE_THREAD_LIMIT");
if (limit_str == null)
return -1;
else
return Integer.decode(limit_str);
}
Map<Integer, Integer>[] pre_results;
final class EvaluatorThread implements Runnable
{
public EvaluatorThread(int start, int stop, int index)
{
this.index = index;
this.start = start;
this.stop = stop;
}
int start, stop, index;
final int map_threads_nr = para;
public void run()
{
Map<Integer, Integer> counts = pre_results[index] = new PDefaultMap<Integer, Integer>(0);
for (int i = start; i < stop; i++) {
Webdoc doc = Generator.documents_array[i];
for (int word_id : doc.words) {
counts.put(Env.canonicalInteger(word_id),
counts.get(word_id) + 1);
}
}
int merge_threads_nr = MERGE_THREAD_LIMIT;
if (merge_threads_nr < 0 || merge_threads_nr > map_threads_nr)
merge_threads_nr = map_threads_nr;
// Now merge!
if (index < merge_threads_nr) { // We get to do some merging!
// First merge pass: linear merge: if reduce threads are bounded,
// merge all on our linear trajectory.
final int max_possible_merge = map_threads_nr / merge_threads_nr;
for (int i = 1; i <= max_possible_merge; i++)
tryMerge(index + (i * merge_threads_nr));
// Second merge pass: log merge.
// Consider the case of 8 threads. We merge as follows:
// #0: 4, 2, 1
// #1: 5, 3
// #2: 6
// #3: 7
// I.e., for d=4,2,1, each thread less than d merges in index+d.
int delta = 0x1;
while ((delta << 1) < merge_threads_nr)
delta <<= 1;
while (delta > 0 && index < delta) {
if (index + delta < merge_threads_nr) // only merge within the reduce-threads block
tryMerge(index + delta);
delta >>= 1;
}
}
// We're done:
doAwait(index);
}
final void
tryMerge(int other_index)
{
if (other_index >= map_threads_nr)
return;
// if (DEBUG)
// System.err.println(" #" + index + " here: trying to merge in #" + other_index);
doAwait(other_index);
final Map<Integer, Integer> counts_a = pre_results[index];
final Map<Integer, Integer> counts_b = pre_results[other_index];
final Set<Map.Entry<Integer, Integer>> entries = counts_b.entrySet();
for (Map.Entry<Integer, Integer> entry : entries) {
counts_a.put(entry.getKey(),
entry.getValue() + counts_a.get(entry.getKey()));
}
// if (DEBUG)
// System.err.println(" #" + index + " merge complete: " + reductor.getAggregate());
}
final void
doAwait(int i)
{
// Notify other threads of completion
try {
barriers[i].await();
} catch (InterruptedException ex) {
return;
} catch (BrokenBarrierException ex) {
return;
}
}
}
public Thread
gen_thread(int index, int start, int stop)
{
return null;
}
public int
get_size() { return Generator.documents_array.length; }
@SuppressWarnings("unchecked")
public final void
compute()
{
compute_prepare();
ExecutorService pool = Executors.newFixedThreadPool(para);
pre_results = (Map<Integer, Integer>[]) new Map[para];
final int total = get_size();
int start = 0;
int size = total / para;
int excess = total - (size * para);
++size;
for (int i = 0; i < para; i++) {
if (excess-- == 0)
--size;
final int stop = start + size;
pool.execute(new EvaluatorThread(start, stop, i));
start = stop;
}
this.result = compute_finish();
pool.shutdown();
}
@Override
public Object
compute_finish()
{
try {
barriers[0].await();
} catch (Exception _ ) {};
return pre_results[0];
}
}