/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tez.dag.history.ats;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.recovery.RecoveryService;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
public class ATSService extends AbstractService {
private static final Log LOG = LogFactory.getLog(ATSService.class);
private LinkedBlockingQueue<DAGHistoryEvent> eventQueue =
new LinkedBlockingQueue<DAGHistoryEvent>();
private final AtomicInteger historyCounter =
new AtomicInteger(0);
private String outputFilePrefix;
private Thread eventHandlingThread;
private AtomicBoolean stopped = new AtomicBoolean(false);
private int eventCounter = 0;
private int eventsProcessed = 0;
private final Object lock = new Object();
public ATSService() {
super(ATSService.class.getName());
}
@Override
public void serviceInit(Configuration conf) throws Exception {
LOG.info("Initializing ATSService");
}
@Override
public void serviceStart() {
LOG.info("Starting ATSService");
eventHandlingThread = new Thread(new Runnable() {
@Override
public void run() {
DAGHistoryEvent event;
while (!stopped.get() && !Thread.currentThread().isInterrupted()) {
// Log the size of the event-queue every so often.
if (eventCounter != 0 && eventCounter % 1000 == 0) {
LOG.info("Event queue stats"
+ ", eventsProcessedSinceLastUpdate=" + eventsProcessed
+ ", eventQueueSize=" + eventQueue.size());
eventCounter = 0;
eventsProcessed = 0;
} else {
++eventCounter;
}
try {
event = eventQueue.take();
} catch (InterruptedException e) {
LOG.info("EventQueue take interrupted. Returning");
return;
}
synchronized (lock) {
++eventsProcessed;
try {
handleEvent(event);
} catch (Exception e) {
// TODO handle failures - treat as fatal or ignore?
LOG.warn("Error handling event", e);
}
}
}
}
}, "HistoryEventHandlingThread");
eventHandlingThread.start();
}
@Override
public void serviceStop() {
LOG.info("Stopping ATSService");
stopped.set(true);
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
}
}
public void handle(DAGHistoryEvent event) {
eventQueue.add(event);
}
private void handleEvent(DAGHistoryEvent event) {
HistoryEventType eventType = event.getHistoryEvent().getEventType();
try {
// TODO integrate with ATS
} catch (Exception e) {
LOG.warn("Could not handle history event, eventType="
+ eventType, e);
// TODO handle error as a fatal event or ignore/skip?
}
}
}