flowNode.addChild(child);
}
}
public void process(PlanNode node) throws DAGOperatorException {
FlowElement newElem = null; // The newly-constructed FlowElement.
FlowElementContext newContext = makeContextForNode(node, mRootSymbolTable);
if (null == node) {
LOG.warn("Null node in plan graph");
return;
} else if (node instanceof OutputNode) {
OutputNode outputNode = (OutputNode) node;
String logicalFlumeNode = outputNode.getFlumeNodeName();
newElem = new OutputElement(newContext,
(Schema) outputNode.getAttr(PlanNode.INPUT_SCHEMA_ATTR),
outputNode.getInputFields(), mFlumeConfig, logicalFlumeNode,
(Schema) outputNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR),
outputNode.getOutputFields(), mRootSymbolTable);
if (null != logicalFlumeNode) {
mLocalFlow.setFlumeRequired(true);
}
} else if (node instanceof MemoryOutputNode) {
MemoryOutputNode memoryNode = (MemoryOutputNode) node;
newElem = new MemoryOutputElement(newContext, memoryNode.getFields());
String bufferName = memoryNode.getName();
// Bind this buffer name to this memory node in the map provided
// by the client.
mMemOutputMap.put(bufferName, (MemoryOutputElement) newElem);
} else if (node instanceof CreateStreamNode) {
// Just perform this operation immediately. Do not translate this into another
// layer. (This results in an empty flow being generated, which is discarded.)
CreateStreamNode createStream = (CreateStreamNode) node;
String streamName = createStream.getName();
StreamSymbol streamSym = new StreamSymbol(createStream);
if (!streamSym.getEventParser().validate(streamSym)) {
// Fails final check of parameters
// TODO: The EventParser is giving better info in its LOG; but this
// should really be communicated back to the user.
throw new DAGOperatorException(
"Stream cannot be created with the specified parameters.");
} else if (mRootSymbolTable.resolve(streamName) != null) {
// TODO: Allow CREATE OR REPLACE STREAM to override this.
throw new DAGOperatorException("Object already exists at top level: " + streamName);
} else {
mRootSymbolTable.addSymbol(streamSym);
mSubmitterSession.sendInfo("CREATE STREAM");
}
if (createStream.getType().equals(StreamSourceType.File)
&& streamSym.getFormatSpec().getParam(FileSourceElement.TIMESTAMP_COL_KEY) == null) {
// We're reading from a file, and making up timestamps based on read time.
// Warn the user that timestamps will change between queries.
StringBuilder sb = new StringBuilder();
sb.append("Warning: File-based streams will set event timestamps based on read time.\n");
sb.append("To specify timestamps explictly, set the ");
sb.append(FileSourceElement.TIMESTAMP_COL_KEY);
sb.append(" event format property.");
mSubmitterSession.sendInfo(sb.toString());
}
} else if (node instanceof DescribeNode) {
// Look up the referenced object in the symbol table and describe it immediately.
DescribeNode describe = (DescribeNode) node;
Symbol sym = mRootSymbolTable.resolve(describe.getIdentifier());
mSubmitterSession.sendInfo(sym.toString());
} else if (node instanceof DropNode) {
// Perform the operation here.
// Remove the objet from our symbol table.
DropNode dropNode = (DropNode) node;
String name = dropNode.getName();
Symbol sym = mRootSymbolTable.resolve(name);
if (null == sym) {
// Shouldn't happen; the type checker already accepted this statement.
throw new DAGOperatorException("No such object at top level: " + name);
}
EntityTarget targetType = dropNode.getType();
// Perform the operation.
mRootSymbolTable.remove(name);
mSubmitterSession.sendInfo("DROP " + targetType.toString().toUpperCase());
} else if (node instanceof NamedSourceNode) {
NamedSourceNode namedInput = (NamedSourceNode) node;
String streamName = namedInput.getStreamName();
Symbol symbol = mRootSymbolTable.resolve(streamName).resolveAliases();
if (null == symbol) {
throw new DAGOperatorException("No symbol for stream: " + streamName);
}
if (!(symbol instanceof StreamSymbol)) {
throw new DAGOperatorException("Identifier " + streamName + " has type: "
+ symbol.getType() + ", not STREAM.");
}
StreamSymbol streamSymbol = (StreamSymbol) symbol;
switch (streamSymbol.getSourceType()) {
case File:
String fileName = streamSymbol.getSource();
newElem = new FileSourceElement(newContext, fileName, streamSymbol.isLocal(),
namedInput.getFields(), streamSymbol);
break;
case Source:
if (!streamSymbol.isLocal()) {
throw new DAGOperatorException("Do not know how to handle a non-local source yet.");
}
String flumeSource = streamSymbol.getSource();
long flowIdNum = mFlowId.getId();
String flowSourceId = "flumebase-flow-" + flowIdNum + "-" + streamSymbol.getName();
newElem = new LocalFlumeSourceElement(newContext, flowSourceId,
mFlumeConfig, flumeSource, (Schema) namedInput.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR),
namedInput.getFields(), streamSymbol);
if (!streamSymbol.isLocal()) {
LOG.info("Created local Flume logical node: " + flowSourceId);
LOG.info("You may need to connect upstream Flume elements to this source.");
}
// Mark Flume as required to execute this flow.
mLocalFlow.setFlumeRequired(true);
break;
case Memory:
newElem = new LocalInMemSourceElement(newContext,
namedInput.getFields(), (InMemStreamSymbol) streamSymbol);
break;
case Node:
String nodeSourceId = "flumebase-flow-" + mFlowId.getId() + "-" + streamSymbol.getName();
newElem = new FlumeNodeElement(newContext, nodeSourceId,
mFlumeConfig, streamSymbol.getSource(),
(Schema) namedInput.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR),
namedInput.getFields(), streamSymbol);
LOG.info("Created local Flume receiver context: " + nodeSourceId);
LOG.info("This will be connected to upstream Flume node: " + streamSymbol.getSource());
// Mark Flume as required to execute this flow.
mLocalFlow.setFlumeRequired(true);
break;
default:
throw new DAGOperatorException("Unhandled stream source type: "
+ streamSymbol.getSourceType());
}
} else if (node instanceof FilterNode) {
FilterNode filterNode = (FilterNode) node;
Expr filterExpr = filterNode.getFilterExpr();
newElem = new FilterElement(newContext, filterExpr);
} else if (node instanceof ProjectionNode) {
ProjectionNode projNode = (ProjectionNode) node;
Schema outSchema = (Schema) projNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR);
newElem = new ProjectionElement(newContext, outSchema, projNode.getInputFields(),
projNode.getOutputFields());
} else if (node instanceof AggregateNode) {
AggregateNode aggNode = (AggregateNode) node;
newElem = new BucketedAggregationElement(newContext, aggNode);
} else if (node instanceof EvaluateExprsNode) {
EvaluateExprsNode evalNode = (EvaluateExprsNode) node;
Schema outSchema = (Schema) evalNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR);
newElem = new EvaluationElement(newContext, evalNode.getExprs(),
evalNode.getPropagateFields(), outSchema);
} else if (node instanceof HashJoinNode) {
HashJoinNode joinNode = (HashJoinNode) node;
newElem = new HashJoinElement(newContext, joinNode);
} else {
throw new DAGOperatorException("Cannot create FlowElement for PlanNode of type: "
+ node.getClass().getName());
}
if (null != newElem) {
// Wrap the FlowElement in a DAGNode.
FlowElementNode elemHolder = new FlowElementNode(newElem);
mapChildren(node, elemHolder);
elemHolder.setId(node.getId());
// Bind the FlowElementNode to the PlanNode.
node.setAttr(LOCAL_FLOW_ELEM_KEY, elemHolder);
if (node.isRoot()) {
// Roots of the plan node => this is a root node in the flow.
mLocalFlow.addRoot(elemHolder);
}
// If we created a BucketedAggregationElement, create its timeout coprocessor.
if (newElem instanceof BucketedAggregationElement) {
BucketedAggregationElement bucketElem = (BucketedAggregationElement) newElem;
FlowElement downstream = getNodeElements(node.getChildren()).get(0).getFlowElement();
FlowElementContext timeoutContext = new DirectCoupledFlowElemContext(downstream);
BucketedAggregationElement.TimeoutEvictionElement timeoutElem =
bucketElem.getTimeoutElement(timeoutContext);
// The timeout element is now upstream to the primary downstream element of the
// BucketedAggregationElement.
downstream.registerUpstream();
timeoutElem.registerUpstream(); // BucketedAggEl't is upstream of the timeout elem.
// Add the timeout element to the BucketedAggregationElement's output list.
// Specify it as the timerElement, since this is a special designation in the
// TimerFlowElemContext.