for (Entry<Schema, List<Path>> schemaEntry :
schemaPaths.entrySet()) {
Schema schema = schemaEntry.getKey();
System.out.println(schema);
InputFormat format = (InputFormat) ReflectionUtils.newInstance(
AvroInputFormat.class, conf);
List<Path> paths = schemaEntry.getValue();
Map<Class<? extends AvroMapper>, List<Path>> mapperPaths
= new HashMap<Class<? extends AvroMapper>, List<Path>>();
// Now, for each set of paths that have a common Schema, build
// a map of Mappers to the paths they're used for
for (Path path : paths) {
Class<? extends AvroMapper> mapperClass = mapperMap.get(path);
if (!mapperPaths.containsKey(mapperClass)) {
mapperPaths.put(mapperClass, new LinkedList<Path>());
}
mapperPaths.get(mapperClass).add(path);
}
// Now each set of paths that has a common InputFormat and Mapper can
// be added to the same job, and split together.
for (Entry<Class<? extends AvroMapper>, List<Path>> mapEntry : mapperPaths
.entrySet()) {
paths = mapEntry.getValue();
Class<? extends AvroMapper> mapperClass = mapEntry.getKey();
if (mapperClass == null) {
mapperClass = (Class<? extends AvroMapper>) conf.getMapperClass();
}
FileInputFormat.setInputPaths(confCopy, paths.toArray(new Path[paths
.size()]));
// Get splits for each input path and tag with InputFormat
// and Mapper types by wrapping in a TaggedInputSplit.
InputSplit[] pathSplits = format.getSplits(confCopy, numSplits);
for (InputSplit pathSplit : pathSplits) {
splits.add(new TaggedInputSplit(pathSplit, conf, format.getClass(),
mapperClass, schema));
}
}
}