/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.nephele.jobmanager.web;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.eclipse.jetty.io.EofException;
import eu.stratosphere.api.common.accumulators.AccumulatorHelper;
import eu.stratosphere.nephele.event.job.AbstractEvent;
import eu.stratosphere.nephele.event.job.ExecutionStateChangeEvent;
import eu.stratosphere.nephele.event.job.JobEvent;
import eu.stratosphere.nephele.event.job.RecentJobEvent;
import eu.stratosphere.nephele.execution.ExecutionState;
import eu.stratosphere.nephele.jobgraph.JobID;
import eu.stratosphere.nephele.jobgraph.JobStatus;
import eu.stratosphere.nephele.jobmanager.JobManager;
import eu.stratosphere.nephele.managementgraph.ManagementGraph;
import eu.stratosphere.nephele.managementgraph.ManagementGraphIterator;
import eu.stratosphere.nephele.managementgraph.ManagementGroupVertex;
import eu.stratosphere.nephele.managementgraph.ManagementGroupVertexID;
import eu.stratosphere.nephele.managementgraph.ManagementVertex;
import eu.stratosphere.nephele.services.accumulators.AccumulatorEvent;
import eu.stratosphere.util.StringUtils;
public class JobmanagerInfoServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
/**
* The log for this class.
*/
private static final Log LOG = LogFactory.getLog(JobmanagerInfoServlet.class);
/**
* Underlying JobManager
*/
private final JobManager jobmanager;
public JobmanagerInfoServlet(JobManager jobmanager) {
this.jobmanager = jobmanager;
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
resp.setStatus(HttpServletResponse.SC_OK);
resp.setContentType("application/json");
try {
if("archive".equals(req.getParameter("get"))) {
writeJsonForArchive(resp.getWriter(), jobmanager.getOldJobs());
}
else if("job".equals(req.getParameter("get"))) {
String jobId = req.getParameter("job");
writeJsonForArchivedJob(resp.getWriter(), jobmanager.getArchive().getJob(JobID.fromHexString(jobId)));
}
else if("groupvertex".equals(req.getParameter("get"))) {
String jobId = req.getParameter("job");
String groupvertexId = req.getParameter("groupvertex");
writeJsonForArchivedJobGroupvertex(resp.getWriter(), jobmanager.getArchive().getJob(JobID.fromHexString(jobId)), ManagementGroupVertexID.fromHexString(groupvertexId));
}
else if("taskmanagers".equals(req.getParameter("get"))) {
resp.getWriter().write("{\"taskmanagers\": " + jobmanager.getNumberOfTaskTrackers() +"}");
}
else if("cancel".equals(req.getParameter("get"))) {
String jobId = req.getParameter("job");
jobmanager.cancelJob(JobID.fromHexString(jobId));
}
else if("updates".equals(req.getParameter("get"))) {
String jobId = req.getParameter("job");
writeJsonUpdatesForJob(resp.getWriter(), JobID.fromHexString(jobId));
}
else{
writeJsonForJobs(resp.getWriter(), jobmanager.getRecentJobs());
}
} catch (Exception e) {
resp.setStatus(HttpServletResponse.SC_BAD_REQUEST);
resp.getWriter().print(e.getMessage());
if (LOG.isWarnEnabled()) {
LOG.warn(StringUtils.stringifyException(e));
}
}
}
/**
* Writes ManagementGraph as Json for all recent jobs
*
* @param wrt
* @param jobs
*/
private void writeJsonForJobs(PrintWriter wrt, List<RecentJobEvent> jobs) {
try {
wrt.write("[");
// Loop Jobs
for (int i = 0; i < jobs.size(); i++) {
RecentJobEvent jobEvent = jobs.get(i);
writeJsonForJob(wrt, jobEvent);
//Write seperator between json objects
if(i != jobs.size() - 1) {
wrt.write(",");
}
}
wrt.write("]");
} catch (EofException eof) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, EofException");
} catch (IOException ioe) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, IOException");
}
}
private void writeJsonForJob(PrintWriter wrt, RecentJobEvent jobEvent) throws IOException {
ManagementGraph jobManagementGraph = jobmanager.getManagementGraph(jobEvent.getJobID());
//Serialize job to json
wrt.write("{");
wrt.write("\"jobid\": \"" + jobEvent.getJobID() + "\",");
wrt.write("\"jobname\": \"" + jobEvent.getJobName()+"\",");
wrt.write("\"status\": \""+ jobEvent.getJobStatus() + "\",");
wrt.write("\"time\": " + jobEvent.getTimestamp()+",");
// Serialize ManagementGraph to json
wrt.write("\"groupvertices\": [");
boolean first = true;
for(ManagementGroupVertex groupVertex : jobManagementGraph.getGroupVerticesInTopologicalOrder()) {
//Write seperator between json objects
if(first) {
first = false;
} else {
wrt.write(","); }
wrt.write(groupVertex.toJson());
}
wrt.write("]");
wrt.write("}");
}
/**
* Writes Json with a list of currently archived jobs, sorted by time
*
* @param wrt
* @param jobs
*/
private void writeJsonForArchive(PrintWriter wrt, List<RecentJobEvent> jobs) {
wrt.write("[");
// sort jobs by time
Collections.sort(jobs, new Comparator<RecentJobEvent>() {
@Override
public int compare(RecentJobEvent o1, RecentJobEvent o2) {
if(o1.getTimestamp() < o2.getTimestamp()) {
return 1;
} else {
return -1;
}
}
});
// Loop Jobs
for (int i = 0; i < jobs.size(); i++) {
RecentJobEvent jobEvent = jobs.get(i);
//Serialize job to json
wrt.write("{");
wrt.write("\"jobid\": \"" + jobEvent.getJobID() + "\",");
wrt.write("\"jobname\": \"" + jobEvent.getJobName()+"\",");
wrt.write("\"status\": \""+ jobEvent.getJobStatus() + "\",");
wrt.write("\"time\": " + jobEvent.getTimestamp());
wrt.write("}");
//Write seperator between json objects
if(i != jobs.size() - 1) {
wrt.write(",");
}
}
wrt.write("]");
}
/**
* Writes infos about archived job in Json format, including groupvertices and groupverticetimes
*
* @param wrt
* @param jobEvent
*/
private void writeJsonForArchivedJob(PrintWriter wrt, RecentJobEvent jobEvent) {
try {
wrt.write("[");
ManagementGraph jobManagementGraph = jobmanager.getManagementGraph(jobEvent.getJobID());
//Serialize job to json
wrt.write("{");
wrt.write("\"jobid\": \"" + jobEvent.getJobID() + "\",");
wrt.write("\"jobname\": \"" + jobEvent.getJobName()+"\",");
wrt.write("\"status\": \""+ jobEvent.getJobStatus() + "\",");
wrt.write("\"SCHEDULED\": "+ jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.SCHEDULED) + ",");
wrt.write("\"RUNNING\": "+ jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.RUNNING) + ",");
wrt.write("\"FINISHED\": "+ jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.FINISHED) + ",");
wrt.write("\"FAILED\": "+ jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.FAILED) + ",");
wrt.write("\"CANCELED\": "+ jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.CANCELED) + ",");
wrt.write("\"CREATED\": " + jobmanager.getArchive().getJobTime(jobEvent.getJobID(), JobStatus.CREATED)+",");
if (jobEvent.getJobStatus() == JobStatus.FAILED) {
ManagementGraphIterator managementGraphIterator = new ManagementGraphIterator(jobManagementGraph,true);
wrt.write("\"failednodes\": [");
HashSet<String> map = new HashSet<String>();
boolean first = true;
while (managementGraphIterator.hasNext()) {
ManagementVertex managementVertex = managementGraphIterator.next();
String instanceName = managementVertex.getInstanceName();
if (managementVertex.getExecutionState() == ExecutionState.FAILED && !map.contains(instanceName)) {
if (first) {
first = false;
} else {
wrt.write(",");
}
wrt.write("{");
wrt.write("\"node\": \"" + instanceName + "\",");
wrt.write("\"message\": \"" + StringUtils.escapeHtml(managementVertex.getOptMessage()) + "\"");
wrt.write("}");
map.add(instanceName);
}
}
wrt.write("],");
}
// Serialize ManagementGraph to json
wrt.write("\"groupvertices\": [");
boolean first = true;
for(ManagementGroupVertex groupVertex : jobManagementGraph.getGroupVerticesInTopologicalOrder()) {
//Write seperator between json objects
if(first) {
first = false;
} else {
wrt.write(","); }
wrt.write(groupVertex.toJson());
}
wrt.write("],");
// write accumulators
AccumulatorEvent accumulators = jobmanager.getAccumulatorResults(jobEvent.getJobID());
Map<String, Object> accMap = AccumulatorHelper.toResultMap(accumulators.getAccumulators());
wrt.write("\n\"accumulators\": [");
int i = 0;
for( Entry<String, Object> accumulator : accMap.entrySet()) {
wrt.write("{ \"name\": \""+accumulator.getKey()+" (" + accumulator.getValue().getClass().getName()+")\","
+ " \"value\": \""+accumulator.getValue().toString()+"\"}\n");
if(++i < accMap.size()) {
wrt.write(",");
}
}
wrt.write("],\n");
wrt.write("\"groupverticetimes\": {");
first = true;
for(ManagementGroupVertex groupVertex : jobManagementGraph.getGroupVerticesInTopologicalOrder()) {
if(first) {
first = false;
} else {
wrt.write(","); }
// Calculate start and end time for groupvertex
long started = Long.MAX_VALUE;
long ended = 0;
// Take earliest running state and latest endstate of groupmembers
for(int j = 0; j < groupVertex.getNumberOfGroupMembers(); j++) {
ManagementVertex vertex = groupVertex.getGroupMember(j);
long running = jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.RUNNING);
if(running != 0 && running < started) {
started = running;
}
long finished = jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.FINISHED);
long canceled = jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.CANCELED);
long failed = jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.FAILED);
if(finished != 0 && finished > ended) {
ended = finished;
}
if(canceled != 0 && canceled > ended) {
ended = canceled;
}
if(failed != 0 && failed > ended) {
ended = failed;
}
}
wrt.write("\""+groupVertex.getID()+"\": {");
wrt.write("\"groupvertexid\": \"" + groupVertex.getID() + "\",");
wrt.write("\"groupvertexname\": \"" + groupVertex + "\",");
wrt.write("\"STARTED\": "+ started + ",");
wrt.write("\"ENDED\": "+ ended);
wrt.write("}");
}
wrt.write("}");
wrt.write("}");
wrt.write("]");
} catch (EofException eof) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, EofException");
} catch (IOException ioe) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, IOException");
}
}
/**
* Writes all updates (events) for a given job since a given time
*
* @param wrt
* @param jobId
*/
private void writeJsonUpdatesForJob(PrintWriter wrt, JobID jobId) {
try {
List<AbstractEvent> events = jobmanager.getEvents(jobId);
//Serialize job to json
wrt.write("{");
wrt.write("\"jobid\": \"" + jobId + "\",");
wrt.write("\"timestamp\": \"" + System.currentTimeMillis() + "\",");
wrt.write("\"recentjobs\": [");
boolean first = true;
for(RecentJobEvent rje: jobmanager.getRecentJobs()) {
if(first) {
first = false;
} else {
wrt.write(","); }
wrt.write("\""+rje.getJobID().toString()+"\"");
}
wrt.write("],");
wrt.write("\"vertexevents\": [");
first = true;
for(AbstractEvent event: events) {
if(event instanceof ExecutionStateChangeEvent) {
if(first) {
first = false;
} else {
wrt.write(","); }
ExecutionStateChangeEvent vertexevent = (ExecutionStateChangeEvent) event;
wrt.write("{");
wrt.write("\"vertexid\": \"" + vertexevent.getVertexID() + "\",");
wrt.write("\"newstate\": \"" + vertexevent.getNewExecutionState() + "\",");
wrt.write("\"timestamp\": \"" + vertexevent.getTimestamp() + "\"");
wrt.write("}");
}
}
wrt.write("],");
wrt.write("\"jobevents\": [");
first = true;
for(AbstractEvent event: events) {
if( event instanceof JobEvent) {
if(first) {
first = false;
} else {
wrt.write(","); }
JobEvent jobevent = (JobEvent) event;
wrt.write("{");
wrt.write("\"newstate\": \"" + jobevent.getCurrentJobStatus() + "\",");
wrt.write("\"timestamp\": \"" + jobevent.getTimestamp() + "\"");
wrt.write("}");
}
}
wrt.write("]");
wrt.write("}");
} catch (EofException eof) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, EofException");
} catch (IOException ioe) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, IOException");
}
}
/**
* Writes infos about one particular archived groupvertex in a job, including all groupmembers, their times and status
*
* @param wrt
* @param jobEvent
* @param groupvertexId
*/
private void writeJsonForArchivedJobGroupvertex(PrintWriter wrt, RecentJobEvent jobEvent, ManagementGroupVertexID groupvertexId) {
try {
ManagementGraph jobManagementGraph = jobmanager.getManagementGraph(jobEvent.getJobID());
ManagementGroupVertex groupvertex = jobManagementGraph.getGroupVertexByID(groupvertexId);
// Serialize ManagementGraph to json
wrt.write("{\"groupvertex\": "+groupvertex.toJson()+",");
wrt.write("\"verticetimes\": {");
boolean first = true;
for(ManagementGroupVertex groupVertex : jobManagementGraph.getGroupVerticesInTopologicalOrder()) {
for(int j = 0; j < groupVertex.getNumberOfGroupMembers(); j++) {
ManagementVertex vertex = groupVertex.getGroupMember(j);
if(first) {
first = false;
} else {
wrt.write(","); }
wrt.write("\""+vertex.getID()+"\": {");
wrt.write("\"vertexid\": \"" + vertex.getID() + "\",");
wrt.write("\"vertexname\": \"" + vertex + "\",");
wrt.write("\"CREATED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.CREATED) + ",");
wrt.write("\"SCHEDULED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.SCHEDULED) + ",");
wrt.write("\"ASSIGNED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.ASSIGNED) + ",");
wrt.write("\"READY\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.READY) + ",");
wrt.write("\"STARTING\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.STARTING) + ",");
wrt.write("\"RUNNING\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.RUNNING) + ",");
wrt.write("\"FINISHING\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.FINISHING) + ",");
wrt.write("\"FINISHED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.FINISHED) + ",");
wrt.write("\"CANCELING\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.CANCELING) + ",");
wrt.write("\"CANCELED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.CANCELED) + ",");
wrt.write("\"FAILED\": "+ jobmanager.getArchive().getVertexTime(jobEvent.getJobID(), vertex.getID(), ExecutionState.FAILED) + "");
wrt.write("}");
}
}
wrt.write("}}");
} catch (EofException eof) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, EofException");
} catch (IOException ioe) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, IOException");
}
}
}