Examples of eu.stratosphere.nephele.jobgraph.JobGraph

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobGraph

eu.stratosphere.nephele.jobgraph.JobGraph
A job graph represents an entire job in Nephele. A job graph must consists at least of one job vertex and must be acyclic.

      inputFile2 = ServerTestUtils.createInputFile(0);
      outputFile1 = new File(ServerTestUtils.getRandomFilename());
      outputFile2 = new File(ServerTestUtils.getRandomFilename());


      // create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));

View Full Code Here


      inputFile1 = ServerTestUtils.createInputFile(0);
      outputFile1 = new File(ServerTestUtils.getRandomFilename());


      // create job graph
      final JobGraph jg = new JobGraph("Self Cross Test Job");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex input = new JobFileInputVertex(inputTaskName, jg);
      input.setFileInputClass(SelfCrossInputTask.class);
      input.setFilePath(new Path(inputFile1.toURI()));

View Full Code Here


      inputFile1 = ServerTestUtils.createInputFile(0);
      outputFile1 = new File(ServerTestUtils.getRandomFilename());


      // create job graph
      final JobGraph jg = new JobGraph("Instance Sharing Test Job");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex input1 = new JobFileInputVertex("Input 1", jg);
      input1.setFileInputClass(FileLineReader.class);
      input1.setFilePath(new Path(inputFile1.toURI()));

View Full Code Here

      inputFile2 = ServerTestUtils.createInputFile(INPUT_DIRECTORY, sizeOfInput);
      outputFile = new File(ServerTestUtils.getTempDir() + File.separator + ServerTestUtils.getRandomFilename());
      jarFile = ServerTestUtils.createJarFile(forwardClassName);


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(new File(testDirectory).toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + forwardClassName + ".jar").toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();

View Full Code Here

      inputFile = ServerTestUtils.createInputFile(0);
      outputFile = new File(ServerTestUtils.getTempDir() + File.separator + ServerTestUtils.getRandomFilename());
      jarFile = ServerTestUtils.createJarFile(exceptionClassName);


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(ExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + exceptionClassName + ".jar")
        .toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);

View Full Code Here

      inputFile = ServerTestUtils.createInputFile(0);
      outputFile = new File(ServerTestUtils.getTempDir() + File.separator + ServerTestUtils.getRandomFilename());
      jarFile = ServerTestUtils.createJarFile(runtimeExceptionClassName);


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(RuntimeExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + runtimeExceptionClassName
        + ".jar").toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);

View Full Code Here

      final File outputFile = new File(ServerTestUtils.getTempDir() + File.separator
        + ServerTestUtils.getRandomFilename());
      final File jarFile = ServerTestUtils.createJarFile(forwardClassName);


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + forwardClassName + ".jar")
        .toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);

View Full Code Here

      jfc.addClass(DoubleSourceTask.class);
      jfc.addClass(DoubleTargetTask.class);
      jfc.createJarFile();


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Double Connection Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input with two Outputs", jg);
      i1.setFileInputClass(DoubleSourceTask.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with two Inputs", jg);
      t1.setTaskClass(DoubleTargetTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i1.connectTo(t1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();

View Full Code Here

      jfc.addClass(DoubleSourceTask.class);
      jfc.addClass(DoubleTargetTask.class);
      jfc.createJarFile();


      // Create job graph
      final JobGraph jg = new JobGraph();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex(jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex(jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();

View Full Code Here

      JarFileCreator jfc = new JarFileCreator(jarFile);
      jfc.addClass(UnionTask.class);
      jfc.createJarFile();


      // Create job graph
      final JobGraph jg = new JobGraph("Union job " + limit);


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));


      // union task
      final JobTaskVertex u1 = new JobTaskVertex("Union", jg);
      u1.setTaskClass(UnionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      u1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(u1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i2.connectTo(u1, ChannelType.IN_MEMORY);
      u1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);


      try {

View Full Code Here

0 1 2 3

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobGraph

eu.stratosphere.client.LocalExecutor

eu.stratosphere.client.program.Client

eu.stratosphere.client.web.JobSubmissionServlet

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.configuration.Configuration

eu.stratosphere.core.fs.FileStatus

eu.stratosphere.core.fs.FileSystem

eu.stratosphere.core.fs.FSDataInputStream

eu.stratosphere.core.fs.Path

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.