Examples of FlowSpecification


Examples of co.cask.cdap.api.flow.FlowSpecification

            responder.sendJson(HttpResponseStatus.BAD_REQUEST, "Must provide a string runnableId for flows/services");
            return;
          }
          runnableId = requestedObj.getRunnableId();
          if (programType == ProgramType.FLOW) {
            FlowSpecification flowSpec = spec.getFlows().get(programId);
            if (flowSpec != null) {
              Map<String, FlowletDefinition> flowletSpecs = flowSpec.getFlowlets();
              if (flowletSpecs != null && flowletSpecs.containsKey(runnableId)) {
                requested = flowletSpecs.get(runnableId).getInstances();
              } else {
                addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                             "Flowlet: " + runnableId + " not found");
View Full Code Here

Examples of co.cask.tigon.api.flow.FlowSpecification

  }

  @Override
  public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    FlowSpecification flowSpec = program.getSpecification();
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
    Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());

    for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) {
      int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances();
      Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances,
                                  "Flowlet %s can have a maximum of %s instances",
                                  flowletDefinition.getFlowletSpec().getName(), maxInstances);
    }
View Full Code Here

Examples of co.cask.tigon.api.flow.FlowSpecification

  @Override
  protected ProgramController launch(Program program, ProgramOptions options,
                                     File hConfFile, File cConfFile, ApplicationLauncher launcher) {
    // Extract and verify parameters
    FlowSpecification flowSpec = program.getSpecification();
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");

    try {
      Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());

      for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) {
        int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances();
        Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances,
                                    "Flowlet %s can have a maximum of %s instances",
                                    flowletDefinition.getFlowletSpec().getName(), maxInstances);
      }

      LOG.info("Configuring flowlets queues");
      Multimap<String, QueueName> flowletQueues = FlowUtils.configureQueue(program, flowSpec, queueAdmin);

      // Launch flowlet program runners
      LOG.info("Launching distributed flow: " + program.getName() + ":" + flowSpec.getName());
      TwillController controller = launcher.launch(new FlowTwillApplication(program, flowSpec,
                                                                            hConfFile, cConfFile, eventHandler));
      DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater(program, controller,
                                                                                                queueAdmin,
                                                                                                flowletQueues);
View Full Code Here

Examples of co.cask.tigon.api.flow.FlowSpecification

      String runIdOption = options.getArguments().getOption(ProgramOptionConstants.RUN_ID);
      Preconditions.checkNotNull(runIdOption, "Missing runId");
      RunId runId = RunIds.fromString(runIdOption);

      FlowSpecification flowSpec = program.getSpecification();
      ProgramType processorType = program.getType();
      Preconditions.checkNotNull(processorType, "Missing processor type.");
      Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");

      String processorName = program.getName();
      Preconditions.checkNotNull(processorName, "Missing processor name.");

      FlowletDefinition flowletDef = flowSpec.getFlowlets().get(flowletName);
      Preconditions.checkNotNull(flowletDef, "Definition missing for flowlet \"%s\"", flowletName);

      Class<?> clz = Class.forName(flowletDef.getFlowletSpec().getClassName(), true,
                                   program.getClassLoader());
      Preconditions.checkArgument(Flowlet.class.isAssignableFrom(clz), "%s is not a Flowlet.", clz);
View Full Code Here

Examples of co.cask.tigon.api.flow.FlowSpecification

    bundler.createBundle(jarLocation, clz);

    Location deployJar = locationFactory.create(clz.getName()).getTempFile(".jar");

    Flow flow = (Flow) clz.newInstance();
    FlowSpecification flowSpec = new DefaultFlowSpecification(clz.getClass().getName(), flow.configure());

    // Creates Manifest
    Manifest manifest = new Manifest();
    manifest.getMainAttributes().put(ManifestFields.MANIFEST_VERSION, "1.0");
    manifest.getMainAttributes().put(ManifestFields.MAIN_CLASS, clz.getName());
View Full Code Here

Examples of com.odiago.flumebase.plan.FlowSpecification

      PlanContext planContext = new PlanContext();
      planContext.setConf(planConf);
      planContext.setSymbolTable(mRootSymbolTable);
      PlanContext retContext = stmt.createExecPlan(planContext);
      msgBuilder.append(retContext.getMsgBuilder().toString());
      FlowSpecification spec = retContext.getFlowSpec();
      if (null != spec) {
        spec.setQuery(query);
        spec.setConf(planConf);
        // Given a flow specification from the AST, run it through
        // necessary post-processing and optimization phases.
        spec.bfs(new PropagateSchemas());
        if (retContext.isExplain()) {
          // We just should explain this flow, but not actually add it.
          msgBuilder.append("Execution plan:\n");
          msgBuilder.append(spec.toString());
          msgBuilder.append("\n");
        } else {
          flowId = addFlow(spec);
        }
      }
View Full Code Here

Examples of com.odiago.flumebase.plan.FlowSpecification

    // Create separate execution plans to gather data from our upstream sources.
    PlanContext leftContext = getSubPlan(leftSrc, planContext);
    PlanContext rightContext = getSubPlan(rightSrc, planContext);

    // Add our upstream source plans to our graph.
    FlowSpecification flowSpec = planContext.getFlowSpec();
    flowSpec.addNodesFromDAG(leftContext.getFlowSpec());
    flowSpec.addNodesFromDAG(rightContext.getFlowSpec());
   
    // Get the true field names that represent keys on the left and right
    // sides of the join.
    String leftName = leftSrc.getSourceName();
    AssignedSymbol leftSym = (AssignedSymbol) getLeftKey().resolveAliases();
    TypedField leftKey = new TypedField(leftSym.getAssignedName(), leftSym.getType());

    String rightName = rightSrc.getSourceName();
    AssignedSymbol rightSym = (AssignedSymbol) getRightKey().resolveAliases();
    TypedField rightKey = new TypedField(rightSym.getAssignedName(), rightSym.getType());

    WindowSpec window = null;
    try {
      // This should evaluate to itself, but make sure to resolve it anyway.
      assert mWindowExpr.isConstant();
      window = (WindowSpec) mWindowExpr.eval(new EmptyEventWrapper());
    } catch (IOException ioe) {
      // mWindowExpr should be constant, so this should be impossible.
      LOG.error("IOException calculating window expression: " + ioe);
      // Signal error by returning a null flow specification anyway.
      planContext.setFlowSpec(null);
      return planContext;
    }

    HashJoinNode joinNode = new HashJoinNode(leftName, rightName, leftKey, rightKey,
        window, getSourceName(), leftContext.getOutFields(), rightContext.getOutFields(),
        planContext.getConf());

    // Set this node to expect multiple input schemas.
    List<Schema> inputSchemas = new ArrayList<Schema>();
    inputSchemas.add(leftContext.getSchema());
    inputSchemas.add(rightContext.getSchema());
    joinNode.setAttr(PlanNode.MULTI_INPUT_SCHEMA_ATTR, inputSchemas);

    flowSpec.attachToLastLayer(joinNode);

    // Create an output context defining our fields, etc.
    PlanContext outContext = new PlanContext(planContext);

    SymbolTable outTable = SymbolTable.mergeSymbols(leftContext.getSymbolTable(),
View Full Code Here

Examples of com.odiago.flumebase.plan.FlowSpecification

    // Create an execution plan for the source(s) of this SELECT stream.
    PlanContext sourceOutCtxt = getSubPlan(source, planContext);
    SymbolTable srcOutSymbolTable = sourceOutCtxt.getSymbolTable();

    // Now incorporate that entire plan into our plan.
    FlowSpecification flowSpec = planContext.getFlowSpec();
    flowSpec.addNodesFromDAG(sourceOutCtxt.getFlowSpec());

    // List of all fields required as output from the source node.
    List<TypedField> allRequiredFields = new ArrayList<TypedField>();

    // All fields carried forward by the aggregation layer from the source layer.
    List<TypedField> groupByPropagateFields = new ArrayList<TypedField>();

    // Another list holds all the fields which the EvaluateExprsNode will need to
    // propagate from the initial source layer forward.
    List<TypedField> exprPropagateFields = new ArrayList<TypedField>();

    // List of all fields with their input names that should be read by the ProjectionNode.
    // This is exprPropagateFields + fields emitted by the expr layer.
    List<TypedField> projectionInputs = new ArrayList<TypedField>();

    // List of all fields returned from the ProjectionNode; this layer
    // uses the translated names from the "x AS y" clauses.
    List<TypedField> projectionOutputs = new ArrayList<TypedField>();

    // Create a list containing the (ordered) set of fields we want emitted to the console.
    List<TypedField> consoleFields = new ArrayList<TypedField>();

    // Populate the field lists defined above
    calculateRequiredFields(srcOutSymbolTable, sourceOutCtxt.getOutFields(),
        allRequiredFields, groupByPropagateFields, exprPropagateFields,
        projectionInputs, projectionOutputs, consoleFields);

    if (where != null) {
      // Non-null filter conditions; apply the filter to all of our sources.
      PlanNode filterNode = new FilterNode(where);
      flowSpec.attachToLastLayer(filterNode);
    }

    // Add an aggregation layer, if required.
    addAggregationToPlan(srcOutSymbolTable, flowSpec, groupByPropagateFields);

    // Evaluate calculated-expression fields.
    addExpressionsToPlan(flowSpec, exprPropagateFields, projectionInputs);

    // Create the projected schema based on the symbol table returned by our source.
    Schema projectedSchema = createFieldSchema(distinctFields(projectionOutputs));
    ProjectionNode projectionNode = new ProjectionNode(projectionInputs, projectionOutputs);
    projectionNode.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, projectedSchema);
    flowSpec.attachToLastLayer(projectionNode);

    if (mHaving != null) {
      // Non-null HAVING conditions; apply another filter to our output.
      PlanNode havingNode = new FilterNode(mHaving);
      flowSpec.attachToLastLayer(havingNode);
    }

    return createReturnedContext(planContext, consoleFields);
  }
View Full Code Here

Examples of com.odiago.flumebase.plan.FlowSpecification

   * Create the output PlanContext that should be returned by createExecPlan().
   */
  private PlanContext createReturnedContext(PlanContext planContext,
      List<TypedField> outputFields) {
    PlanContext outContext = planContext;
    FlowSpecification flowSpec = planContext.getFlowSpec();
    if (planContext.isRoot()) {
      String selectTarget = planContext.getConf().get(CLIENT_SELECT_TARGET_KEY,
          DEFAULT_CLIENT_SELECT_TARGET);
      if (CONSOLE_SELECT_TARGET.equals(selectTarget)) {
        // SELECT statements that are root queries go to the output node.

        // This output node may emit Avro records to a Flume node. These records
        // should use more user-friendly names for the fields than the anonymized
        // field names we use internally. Create a final schema for the output
        // plan node.
        String outputName = getOutputName();
        List<TypedField> outSchemaFields = new ArrayList<TypedField>();
        List<TypedField> distinctOutFields = distinctFields(outputFields);
        for (TypedField outField : distinctOutFields) {
          String safeName = avroSafeName(outField.getDisplayName());
          outSchemaFields.add(new TypedField(safeName, outField.getType()));
        }
        Schema finalSchema = createFieldSchema(outSchemaFields, outputName);
        OutputNode outputNode = new OutputNode(outputFields, outSchemaFields, outputName);
        outputNode.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, finalSchema);
        flowSpec.attachToLastLayer(outputNode);
      } else {
        // Client has specified that outputs of this root query go to a named memory buffer.
        flowSpec.attachToLastLayer(new MemoryOutputNode(selectTarget,
            distinctFields(outputFields)));
      }
    } else {
      // If the initial projection contained both explicitly selected fields as
      // well as implicitly selected fields (e.g., for the WHERE clause), attach another
      // projection layer that extracts only the explicitly selected fields.

      // SELECT as a sub-query needs to create an output context with a
      // symbol table that contains the fields we expose through projection.
      // We also need to set the output field names and output schema in our
      // returned context.
      outContext = new PlanContext(planContext);
      SymbolTable inTable = planContext.getSymbolTable();
      SymbolTable outTable = new HashSymbolTable(inTable);
      outputFields = distinctFields(outputFields);
      outTable.addAll(mFieldSymbols);
      Schema outputSchema = createFieldSchema(outputFields);
      ProjectionNode cleanupProjection = new ProjectionNode(outputFields, outputFields);
      cleanupProjection.setAttr(PlanNode.OUTPUT_SCHEMA_ATTR, outputSchema);
      flowSpec.attachToLastLayer(cleanupProjection);

      outContext.setSymbolTable(outTable);
      outContext.setSchema(outputSchema);
      outContext.setOutFields(outputFields);
    }
View Full Code Here

Examples of com.odiago.flumebase.plan.FlowSpecification

    // Create an execution plan to build the source (it may be a single node
    // representing a Flume source or file, or it may be an entire DAG because
    // we use another SELECT statement as a source) inside a new context.
    PlanContext sourceInCtxt = new PlanContext(planContext);
    sourceInCtxt.setRoot(false);
    sourceInCtxt.setFlowSpec(new FlowSpecification(planContext.getConf()));
    return subStmt.createExecPlan(sourceInCtxt);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.