// ordered by score, getting passed per list to the PreFetchBuffer. This will generate PreFetchDatums that contain a key
// based on the hash of the IP address (with a range of values == number of reducers), plus a list of URLs and a target
// crawl time.
Pipe prefetchPipe = new GroupBy("Distributing URL sets", splitter.getRHSPipe(), GroupedUrlDatum.getGroupingField(), ScoredUrlDatum.getSortingField(), true);
prefetchPipe = new Every(prefetchPipe, new MakeFetchSetsBuffer(fetchJobPolicy, numReducers), Fields.RESULTS);
Pipe fetchPipe = new GroupBy("Fetching URL sets", prefetchPipe, FetchSetDatum.getGroupingField(), FetchSetDatum.getSortingField());
fetchPipe = new Every(fetchPipe, new FetchBuffer(fetcher), Fields.RESULTS);
Pipe fetchedContent = new Pipe(CONTENT_PIPE_NAME, new Each(fetchPipe, new FilterErrorsFunction()));