package org.xadoop.servlet.actions;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import org.xadoop.XadoopConfig;
import org.xadoop.XadoopConstants;
import org.xadoop.hadoop.HadoopConfFile;
import org.xadoop.runtime.SystemCallRunnable;
import org.xadoop.servlet.actions.results.AbstractResult;
import org.xadoop.servlet.actions.results.Failure;
import org.xadoop.servlet.actions.results.Success;
import org.xadoop.servlet.pages.XQueryMRPage;
import org.xadoop.servlet.utils.PostRequest;
import org.xadoop.xquerymr.Pipeline;
import org.xadoop.xquerymr.XQueryMRConfFile;
import org.xadoop.xquerymr.XQueryMRSaxonConfFile;
import org.xadoop.xquerymr.XQueryMRZorbaConfFile;
/**
* An action to execute XQueryMR.
*
* @author Lukas Blunschi
*
*/
public class XQueryMRAction extends AbstractAction {
@SuppressWarnings("unused")
private static void copyFile(File sourceFile, File destFile) throws IOException {
if (!destFile.exists()) {
destFile.createNewFile();
}
FileChannel source = null;
FileChannel destination = null;
try {
source = new FileInputStream(sourceFile).getChannel();
destination = new FileOutputStream(destFile).getChannel();
destination.transferFrom(source, 0, source.size());
}
catch (IOException e) {
e.printStackTrace();
}
finally {
if (source != null) {
source.close();
}
if (destination != null) {
destination.close();
}
}
}
public static final String ACTIONNAME = "xquerymr";
public static final String FILENAME_QUERY = "prolog.xq";
public static final String FILENAME_DATA = "input-data.xml";
public static final String FILENAME_CONF = "conf.xml";
public static final String FILENAME_OUTPUT = "output.txt";
public static final DateFormat execDirFormat = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
public AbstractResult execute(HttpServletRequest req) {
// upload path
ServletContext ctx = req.getSession().getServletContext();
String pathUploadDir = (String) ctx.getAttribute(XadoopConstants.CTX_PATHUPLOADDIR);
XadoopConfig config = (XadoopConfig) ctx.getAttribute(XadoopConstants.CTX_CONFIG);
// get required parameters
PostRequest postReq = new PostRequest();
try {
postReq.parse(req, pathUploadDir, true);
} catch (Exception e) {
return new Failure(e.getMessage());
}
String xqueryengine = postReq.getFormField(XQueryMRPage.XMLPARSER);
String querytext = postReq.getFormField(XQueryMRPage.QUERYTEXT);
String streamingEnabled = postReq.getFormField(XQueryMRPage.STREAMING_SUPPORT);
String datatext = postReq.getFormField(XQueryMRPage.DATATEXT);
String dataExists = postReq.getFormField(XQueryMRPage.DATAEXISTS);
String datafile = postReq.getFormField(XQueryMRPage.DATAFILE);
String datapath = postReq.getFormField(XQueryMRPage.DATAPATH);
String numTasksMapStr = postReq.getFormField(XQueryMRPage.NUMTASKSMAP);
String numTasksRedStr = postReq.getFormField(XQueryMRPage.NUMTASKSRED);
// execution directory
Date now = new Date();
String pathExecDir = pathUploadDir + execDirFormat.format(now) + "/";
File execDir = new File(pathExecDir);
execDir.mkdir();
// query
if (querytext.trim().length() > 0) {
File queryFile = new File(pathExecDir + FILENAME_QUERY);
try {
FileWriter writer = new FileWriter(queryFile);
writer.write(querytext);
writer.close();
} catch (IOException e) {
return new Failure("Failure writing query file: " + e.getMessage());
}
}
// data
// if text is given use text,
// otherwise test if path to file is given,
// otherwise test for a given file
if (datatext.trim().length() > 0) {
File dataFile = new File(pathExecDir + FILENAME_DATA);
try {
FileWriter writer = new FileWriter(dataFile);
writer.write(datatext);
writer.close();
} catch (IOException e) {
return new Failure("Failure writing data file: " + e.getMessage());
}
//if path to input data is given
} else if (datapath.trim().length() > 0) {
//if the data does not yet exist in the system
if (!dataExists.equals("true")) {
Runtime rt = Runtime.getRuntime();
try {
//create a symbolic link
rt.exec("ln -s "+datapath.trim()+" "+pathExecDir+FILENAME_DATA);
} catch (IOException e) {
System.err.println(e.getMessage());
}
}
//if input file is provided via browser mask
} else if (datafile != null) {
File dataFileOrg = new File(pathUploadDir + datafile);
File dataFileDst = new File(pathExecDir + FILENAME_DATA);
if (!dataFileOrg.renameTo(dataFileDst)) {
return new Failure("Could not move data file to execution directory.");
}
} else if (dataExists.equals("false")) {
return new Failure("No data given.");
}
// number of tasks
int numTasksMap = 1;
int numTasksReduce = 1;
if (numTasksMapStr != null) {
try {
numTasksMap = Integer.parseInt(numTasksMapStr);
} catch (Exception e) {
}
}
if (numTasksRedStr == null) {
numTasksReduce = numTasksMap;
} else {
try {
numTasksReduce = Integer.parseInt(numTasksRedStr);
} catch (Exception e) {
}
}
// conf file
Pipeline pipeline = null;
HadoopConfFile confFileObj = null;
if (xqueryengine.equals("zorba")) {
pipeline = new Pipeline(querytext, XadoopConstants.PIPELINE_MAPPREFIX);
XQueryMRZorbaConfFile confFileObjTmp = new XQueryMRZorbaConfFile(config);
confFileObjTmp.setPipeline(pipeline.toString());
System.out.println(pipeline.toString());
confFileObjTmp.setQueryFile("/" + FILENAME_QUERY);
confFileObj = confFileObjTmp;
} else if (xqueryengine.equals("saxon")){
pipeline = new Pipeline(querytext, XadoopConstants.SAXON_PIPELINE_MAPPREFIX);
XQueryMRSaxonConfFile confFileObjTmp = new XQueryMRSaxonConfFile(config);
confFileObjTmp.setPipeline(pipeline.toString());
confFileObjTmp.setProlog("/" + FILENAME_QUERY);
confFileObjTmp.setStreamEnabled(streamingEnabled);
confFileObj = confFileObjTmp;
} else {
pipeline = new Pipeline(querytext, XadoopConstants.XQUERYMR_PIPELINE_MAPPREFIX);
XQueryMRConfFile confFileObjTmp = new XQueryMRConfFile();
confFileObjTmp.setPipeline(pipeline.toString());
confFileObjTmp.setProlog("/" + FILENAME_QUERY);
confFileObj = confFileObjTmp;
}
int numMapTasks = numTasksMap;
int numReduceTasks = numTasksReduce;
confFileObj.setMapredMapTasks(numMapTasks);
confFileObj.setMapredReduceTasks(numReduceTasks);
File confFile = new File(pathExecDir + FILENAME_CONF);
try {
FileWriter writer = new FileWriter(confFile);
writer.write(confFileObj.toXml());
writer.close();
} catch (IOException e) {
return new Failure("Failure writing conf file: " + e.getMessage());
}
// commands
String pathJar = null;
if (xqueryengine.equals("zorba")) {
pathJar = "../../dist/xadoop-" + XadoopConstants.VERSION + ".jar";
} else if (xqueryengine.equals("saxon")) {
pathJar = "../../dist/saxon_driver.jar";
} else {
pathJar = "../../xquerymr/XQueryMR.jar";
}
String pathHadoopBin = config.getHadoopBinDir();
List<String> commands = new ArrayList<String>();
//clean up HDFS
if (!dataExists.equals("true"))
commands.add(pathHadoopBin + "hadoop fs -rm /" + FILENAME_DATA);
commands.add(pathHadoopBin + "hadoop fs -rm /" + FILENAME_QUERY);
commands.add(pathHadoopBin + "hadoop fs -rmr /output");
//load input data into Hadoop
if (!dataExists.equals("true"))
commands.add(pathHadoopBin + "hadoop fs -copyFromLocal " + FILENAME_DATA + " /");
commands.add(pathHadoopBin + "hadoop fs -copyFromLocal " + FILENAME_QUERY + " /");
//run hadoop
commands.add(pathHadoopBin + "hadoop jar " + pathJar + " -conf " + FILENAME_CONF + " /" + FILENAME_DATA
+ " /output");
//remove old output
commands.add("rm -rf output/");
commands.add(pathHadoopBin + "hadoop fs -copyToLocal /output .");
// execute
String pathOutputFile = pathExecDir + FILENAME_OUTPUT;
SystemCallRunnable runnable = new SystemCallRunnable(pathOutputFile, commands, execDir);
new Thread(runnable, "xquerymr").start();
return new Success("&" + XQueryMRPage.EXECDIR + "=" + pathExecDir);
}
}