if (shared) {
if (!fs.exists(libjarsDir)) {
FileSystem.mkdirs(fs, libjarsDir, mapredSysPerms);
}
MD5Hash md5hash = MD5Hash.digest(new
FileInputStream(originalJarPath.toUri().getPath()));
uploadJarPath = copyRemoteFiles(fs, libjarsDir, originalJarPath, job,
replication, md5hash.toString());
URI pathURI = new URI(uploadJarPath.toUri().toString());
DistributedCache.addSharedArchiveToClassPath(uploadJarPath, job);
fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
md5hash.getFileLength(), 0));
} else {
// Otherwise we copy jar to JT's filesystem
uploadJarPath = new Path(uploadFileDir, "job.jar");
fs.copyFromLocalFile(originalJarPath, uploadJarPath);
}
} catch (URISyntaxException ue) {
// should not throw an uri exception
throw new IOException("Failed to create uri for " + originalJar);
}
job.setJar(uploadJarPath.toString());
fs.setReplication(uploadJarPath, replication);
try {
fs.setPermission(uploadJarPath, new FsPermission(JOB_FILE_PERMISSION));
} catch (IOException ioe) {
LOG.warn("Unable to set job jar permission");
}
} else {
LOG.warn("No job jar file set. User classes may not be found. "+
"See JobConf(Class) or JobConf#setJar(String).");
}
// add all the command line files/ jars and archive
// first copy them to jobtrackers filesystem
if (files != null) {
if (!fs.exists(filesDir)) {
FileSystem.mkdirs(fs, filesDir, mapredSysPerms);
}
String[] fileArr = files.split(",");
for (String tmpFile: fileArr) {
Path tmp = new Path(tmpFile);
Path newPath;
FileStatus fStatus = null;
MD5Hash md5hash = null;
try {
if (shared) {
md5hash
= MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
newPath = copyRemoteFiles(fs, filesDir, tmp, job, replication,
md5hash.toString());
URI pathURI = new URI(newPath.toUri().toString() + "#" +
newPath.getName());
DistributedCache.addSharedCacheFile(pathURI, job);
fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
md5hash.getFileLength(),
0));
} else {
newPath = copyRemoteFiles(fs, filesDir, tmp, job, replication);
fStatus = DistributedCache.getFileStatus(job, newPath.toUri());
URI pathURI = new URI(newPath.toUri().toString() + "#" +
newPath.getName());
DistributedCache.addCacheFile(pathURI, job);
fileInfo.put(pathURI, new FileInfo(null,
fStatus.getLen(),
fStatus.getModificationTime()));
}
} catch(URISyntaxException ue) {
//should not throw a uri exception
throw new IOException("Failed to create uri for " + tmpFile);
}
DistributedCache.createSymlink(job);
}
}
if (libjars != null) {
if (!fs.exists(libjarsDir)) {
FileSystem.mkdirs(fs, libjarsDir, mapredSysPerms);
}
String[] libjarsArr = libjars.split(",");
for (String tmpjars: libjarsArr) {
Path tmp = new Path(tmpjars);
Path newPath;
if (shared) {
MD5Hash md5hash
= MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
newPath = copyRemoteFiles(fs, libjarsDir, tmp, job, replication,
md5hash.toString());
DistributedCache.addSharedArchiveToClassPath(newPath, job);
fileInfo.put(newPath.makeQualified(newPath.getFileSystem(job)).toUri(),
new FileInfo(md5hash.toString(),
md5hash.getFileLength(),
0));
} else {
newPath = copyRemoteFiles(fs, libjarsDir, tmp, job, replication);
DistributedCache.addArchiveToClassPath(newPath, job);
FileStatus fStatus = DistributedCache.getFileStatus(job,
newPath.toUri());
fileInfo.put(newPath.makeQualified(newPath.getFileSystem(job)).toUri(),
new FileInfo(null,
fStatus.getLen(),
fStatus.getModificationTime()));
}
}
}
if (archives != null) {
if (!fs.exists(archivesDir)) {
FileSystem.mkdirs(fs, archivesDir, mapredSysPerms);
}
String[] archivesArr = archives.split(",");
for (String tmpArchives: archivesArr) {
Path tmp = new Path(tmpArchives);
Path newPath;
MD5Hash md5hash = null;
FileStatus fStatus = null;
try {
if (shared) {
md5hash
= MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication,
md5hash.toString());
URI pathURI = new URI(newPath.toUri().toString() + "#" +
newPath.getName());
DistributedCache.addSharedCacheArchive(pathURI, job);
fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
md5hash.getFileLength(),
0));
} else {
newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication);
fStatus = DistributedCache.getFileStatus(job, newPath.toUri());