package com.alipay.bluewhale.core.task.common;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.log4j.Logger;
import backtype.storm.Config;
import backtype.storm.generated.Bolt;
import backtype.storm.generated.ComponentObject;
import backtype.storm.generated.Grouping;
import backtype.storm.generated.JavaObject;
import backtype.storm.generated.ShellComponent;
import backtype.storm.generated.SpoutSpec;
import backtype.storm.generated.StateSpoutSpec;
import backtype.storm.generated.StormTopology;
import backtype.storm.spout.ISpout;
import backtype.storm.spout.ShellSpout;
import backtype.storm.task.IBolt;
import backtype.storm.task.ShellBolt;
import backtype.storm.task.TopologyContext;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.utils.TimeCacheMap;
import backtype.storm.utils.Utils;
import com.alipay.bluewhale.core.callback.RunnableCallback;
import com.alipay.bluewhale.core.cluster.StormConfig;
import com.alipay.bluewhale.core.messaging.IConnection;
import com.alipay.bluewhale.core.stats.BaseTaskStatsRolling;
import com.alipay.bluewhale.core.stats.Stats;
import com.alipay.bluewhale.core.task.error.ITaskReportErr;
import com.alipay.bluewhale.core.task.executer.BoltExecutors;
import com.alipay.bluewhale.core.task.executer.SpoutExecutors;
import com.alipay.bluewhale.core.task.group.MkGrouper;
import com.alipay.bluewhale.core.task.transfer.TaskSendTargets;
import com.alipay.bluewhale.core.thrift.Thrift;
import com.alipay.bluewhale.core.utils.StormUtils;
import com.alipay.bluewhale.core.utils.TimeUtils;
import com.alipay.bluewhale.core.work.transfer.WorkerTransfer;
/**
* task�Ĺ��÷���
*
* @author yannian
*
*/
public class TasksCommon {
private final static Logger LOG = Logger.getLogger(TasksCommon.class);
public static BaseTaskStatsRolling mk_task_stats(Object obj, int rate) {
BaseTaskStatsRolling stat=null;
if (obj instanceof IBolt) {
stat= Stats.mk_bolt_stats(rate);
}else{
stat=Stats.mk_spout_stats(rate);
}
return stat;
}
public static Long tuple_time_delta(TimeCacheMap start_times, Tuple tuple) {
Object start_time = start_times.remove(tuple);
if (start_time != null) {
return TimeUtils.time_delta_ms((Long) start_time);
}
return null;
}
public static Long tuple_time_delta(Map start_times, Tuple tuple) {
Object start_time = start_times.remove(tuple);
if (start_time != null) {
return TimeUtils.time_delta_ms((Long) start_time);
}
return null;
}
public static void put_xor(TimeCacheMap pending, Object key, Object id) {
//synchronized (pending) {
Object curr = new Integer(0);
if (pending.containsKey(key)) {
curr = pending.get(key);
}
pending.put(key, StormUtils.bit_xor(curr, id));
//}
}
public static void put_xor(Map pending, Object key, Object id) {
//synchronized (pending) {
Object curr = new Integer(0);
if (pending.containsKey(key)) {
curr = pending.get(key);
}
pending.put(key, StormUtils.bit_xor(curr, id));
//}
}
/**
* ��ȡָ��component_id�϶�Ӧ��task-object
* @param topology
* @param component_id
* @return
*/
public static Object get_task_object(StormTopology topology,
String component_id) {
Map<String, SpoutSpec> spouts = topology.get_spouts();
Map<String, Bolt> bolts = topology.get_bolts();
Map<String, StateSpoutSpec> state_spouts = topology.get_state_spouts();
ComponentObject obj = null;
if (spouts.containsKey(component_id)) {
obj = spouts.get(component_id).get_spout_object();
}else if (bolts.containsKey(component_id)) {
obj = bolts.get(component_id).get_bolt_object();
}else if (state_spouts.containsKey(component_id)) {
obj = state_spouts.get(component_id).get_state_spout_object();
}
if (obj == null) {
LOG.error("get_task_object->obj == null");
throw new RuntimeException("Could not find " + component_id
+ " in " + topology.toString());
}
Object componentObject = getSetComponentObject(obj);
Object rtn=null;
if (componentObject instanceof JavaObject) {
rtn=Thrift.instantiateJavaObject((JavaObject) componentObject);
}else if (componentObject instanceof ShellComponent) {
if (spouts.containsKey(component_id)) {
rtn=new ShellSpout((ShellComponent) componentObject);
} else {
rtn= new ShellBolt((ShellComponent) componentObject);
}
}else{
rtn=componentObject;
}
return rtn;
}
/**
* ��ȡ Task Object
* @param obj
* @return
*/
private static Object getSetComponentObject(ComponentObject obj) {
if (obj.getSetField() == ComponentObject._Fields.SERIALIZED_JAVA) {
return Utils.deserialize(obj.get_serialized_java());
} else if (obj.getSetField() == ComponentObject._Fields.JAVA_OBJECT) {
return obj.get_java_object();
} else {
return obj.get_shell();
}
}
// ��ȡ��ǰtask��ÿ��streamӦ��������ЩcommponID,�Լ���������η����
public static Map<String, Map<String, MkGrouper>> outbound_components(
TopologyContext topology_context) {
Map<String, Map<String, MkGrouper>> rr = new HashMap<String, Map<String, MkGrouper>>();
// <Stream_id,<component,Grouping>>
Map<String, Map<String, Grouping>> output_groupings = topology_context
.getThisTargets();
for (Entry<String, Map<String, Grouping>> entry : output_groupings
.entrySet()) {
Map<String, Grouping> component_grouping = entry.getValue();
String stream_id = entry.getKey();
Fields out_fields = topology_context.getThisOutputFields(stream_id);
Map<String, MkGrouper> componentGrouper = new HashMap<String, MkGrouper>();
for (Entry<String, Grouping> cg : component_grouping.entrySet()) {
String component = cg.getKey();
Grouping tgrouping = cg.getValue();
int num_tasks = topology_context.getComponentTasks(component)
.size();
if (num_tasks > 0) {
MkGrouper grouper = new MkGrouper(out_fields, tgrouping,
num_tasks);
componentGrouper.put(component, grouper);
}
}
if (componentGrouper.size() > 0) {
rr.put(stream_id, componentGrouper);
}
}
return rr;
}
// get-readable-name
public static String get_readable_name(TopologyContext topology_context) {
return topology_context.getThisComponentId();
}
/**
* ��ȡcomponent��Ӧ��������Ϣ
* @param storm_conf
* @param topology_context
* @param component_id
* @return
*/
@SuppressWarnings("unchecked")
public static Map component_conf(Map storm_conf,
TopologyContext topology_context, String component_id) {
List<Object> to_remove = StormConfig.All_CONFIGS();
to_remove.remove(Config.TOPOLOGY_DEBUG);
to_remove.remove(Config.TOPOLOGY_MAX_SPOUT_PENDING);
to_remove.remove(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
to_remove.remove(Config.TOPOLOGY_TRANSACTIONAL_ID);
Map<Object,Object> spec_conf = new HashMap<Object,Object>();
String jconf = topology_context.getComponentCommon(component_id)
.get_json_conf();
if (jconf != null) {
spec_conf = (Map<Object,Object>) StormUtils.from_json(jconf);
}
for (Object p : to_remove) {
spec_conf.remove(p);
}
spec_conf.putAll(storm_conf);
return spec_conf;
}
public static RunnableCallback mk_executors(Object _task_obj,
WorkerTransfer transfer_fn, Map _storm_conf, IConnection _puller,
TaskSendTargets sendTargets, AtomicBoolean _storm_active_atom,
TopologyContext _topology_context, TopologyContext _user_context,
BaseTaskStatsRolling _task_stats, ITaskReportErr _report_error) {
if (_task_obj instanceof IBolt) {
return new BoltExecutors((IBolt) _task_obj, transfer_fn,
_storm_conf, _puller, sendTargets, _storm_active_atom,
_topology_context, _user_context, _task_stats,
_report_error);
}
if (_task_obj instanceof ISpout) {
return new SpoutExecutors((ISpout) _task_obj, transfer_fn,
_storm_conf, _puller, sendTargets, _storm_active_atom,
_topology_context, _user_context, _task_stats);
}
return null;
}
}