Examples of Flow


Examples of azkaban.flow.Flow

      List<String> rootJobs = manager.getRootNamesByFolder(folder);
      Collections.sort(rootJobs);

      JSONArray rootJobObj = new JSONArray();
      for (String root: rootJobs) {
        Flow flow = manager.getFlow(root);
        JSONObject flowObj = getJSONDependencyTree(flow);
        rootJobObj.add(flowObj);
      }
     
      return rootJobObj.toJSONString();
View Full Code Here

Examples of azkaban.workflow.Flow

            page.render();
            return;
          }
         
          // This will be used the other
          Flow displayFlow = new Flow(flow.getName(), (Props)null);
          fillFlow(displayFlow, flow);
          displayFlow.validateFlow();
         
          String flowJSON = createJsonFlow(displayFlow);
          page.add("jsonflow", flowJSON);
          page.add("action", "run");
          page.add("joblist", createJsonJobList(displayFlow));
        }
        else if (hasParam(req, "id")) {
          long id = Long.parseLong(getParam(req, "id"));
             FlowExecutionHolder holder = allFlows.loadExecutableFlow(id);
          ExecutableFlow executableFlow = holder.getFlow();

          // This will be used the other
          Flow displayFlow = new Flow(executableFlow.getName(), (Props)null);
          fillFlow(displayFlow, executableFlow);
          displayFlow.validateFlow();
         
          String flowJSON = createJsonFlow(displayFlow);
          page.add("jsonflow", flowJSON);
          page.add("id", id);
          if (executableFlow.getStartTime() != null) {
View Full Code Here

Examples of be.isencia.passerelle.model.Flow

              // Fixing Checkstyle bug JC Pret
                flow = createFlow(step, false);
                // End JC Pret
            }
            ***/
            Flow flow = createFlow(step, false);
            // End Complete if...else block commented out...

            // Bug 17641
            // displayViewComp(flow, step);

            try {
                if (StepType.ACTOR.equals(step.getType())
                        || useBossaNovaDirector) {
                    flow.setDirector(batch.getBatchDirector());
                } else {
                    Director dir = originalDirectors.get(step);
                    flow.setDirector(dir);
                }
                for (int i = 0; i < step.getIterationCount(); i++) {
                    BossaNovaData.getSingleton().getApplication().getTraceComponent().trace(
                            this.getBatchDirector(),
                            "Executing " + step.getName() + "("
                            + (currentRunningStepIndex + 1)
                            + ") - iteration " + (i + 1)
                            + " on " + step.getIterationCount());
                    // if stop is requested we need to stop execution
                    if (isStopRequested()) {
                        flow.setDirector(null);
                        return;
                    }
                    //application.setSelectedStep(step, true);
                    BossaNovaData.getSingleton().getApplication().launchModel(flow, modelExecutor, true, false);
                    // if stop is requested we need to stop execution
                    if (isStopRequested()) {
                        flow.setDirector(null);
                        return;
                    }
                }
                flow.setDirector(null);
            } catch (Exception e) {
                e.printStackTrace();
                step.setOnFault(true);
            }
        } else {
View Full Code Here

Examples of cascading.flow.Flow

    }

    @SuppressWarnings("rawtypes")
    @Override
    public Flow connect(FlowDef flowDef) {
        final Flow flow = super.connect(flowDef);
        Flow proxy = (Flow) Proxy.newProxyInstance(Flow.class.getClassLoader(), new Class[] { Flow.class }, new InvocationHandler() {
            @Override
            public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
                        Object invocation = ReflectionUtils.invoke(method, flow, args);
                        if ("complete".equals(method.getName())) {
                            printStats(flow);
View Full Code Here

Examples of cascading.flow.Flow

public abstract class StatsUtils {

    @SuppressWarnings("rawtypes")
    public static Flow proxy(final Flow flow) {
        Flow proxy = (Flow) Proxy.newProxyInstance(Flow.class.getClassLoader(), new Class[] { Flow.class }, new InvocationHandler() {
            @Override
            public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
                        Object invocation = ReflectionUtils.invoke(method, flow, args);
                        if ("complete".equals(method.getName())) {
                            printStats(flow);
View Full Code Here

Examples of cascading.flow.Flow

            Pipe importPipe = new Each("url importer", new Fields("line"), new CreateUrlFromTextFunction(new SimpleUrlNormalizer(), new SimpleUrlValidator()));

            Tap urlSink = _platform.makeTap(_platform.makeBinaryScheme(CrawlDbDatum.FIELDS), _destDirPath, SinkMode.REPLACE);

            FlowConnector flowConnector = _platform.makeFlowConnector();
            Flow flow = flowConnector.connect(urlSource, urlSink, importPipe);
            flow.complete();
        } catch (Exception e) {
            throw e;
        }
    }
View Full Code Here

Examples of cascading.flow.Flow

                BasePath curLoopDirPath = CrawlDirUtils.makeLoopDir(platform, outputPath, curLoop);
                String curLoopDirName = curLoopDirPath.getName();
                setLoopLoggerFile(logsDir+curLoopDirName, curLoop);

                Flow flow = DemoCrawlWorkflow.createFlow(curLoopDirPath, crawlDbPath, defaultPolicy, userAgent, urlFilter, options);
                flow.complete();
               
                // Writing out .dot files is a good way to verify your flows.
//              flow.writeDOT("build/valid-flow.dot");

                // Update crawlDbPath to point to the latest crawl db
View Full Code Here

Examples of cascading.flow.Flow

                        new Fields(UrlDatum.URL_FN));
        resultsPipe = new Every(resultsPipe, new LatestUrlDatumBuffer(), Fields.RESULTS);


        FlowConnector flowConnector = platform.makeFlowConnector();
        Flow flow = flowConnector.connect(sources, resultSink, resultsPipe);
        flow.complete();
       
        // verify that the resulting pipe has the latest tuple
       
        Tap testSink = platform.makeTap(platform.makeBinaryScheme(UrlDatum.FIELDS), resultsPath);
        TupleEntryIterator reader = testSink.openForRead(platform.makeFlowProcess());
View Full Code Here

Examples of cascading.flow.Flow

            sinkMap.put(writableSeqFileDataPipe.getName(), writableSeqFileSink);
            tailPipes.add(writableSeqFileDataPipe);
        }
       
        FlowConnector flowConnector = platform.makeFlowConnector();
        Flow flow = flowConnector.connect(inputSource, sinkMap, tailPipes.toArray(new Pipe[tailPipes.size()]));

        return flow;
    }
View Full Code Here

Examples of cascading.flow.Flow

            BasePath inputPath = platform.makePath(input);
            Tap sourceTap = platform.makeTap(platform.makeTextScheme(), inputPath);
            Tap sinkTap = platform.makeTap(platform.makeBinaryScheme(UrlDatum.FIELDS), crawlDBPath, SinkMode.REPLACE);
           
            FlowConnector flowConnector = platform.makeFlowConnector();
            Flow flow = flowConnector.connect(sourceTap, sinkTap, importPipe);
            flow.complete();
        }

        return crawlDBPath.getAbsolutePath();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.