Examples of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

eu.stratosphere.nephele.jobgraph.JobFileInputVertex
A JobFileInputVertex is a specific subtype of a {@link AbstractJobInputVertex} and is designedfor Nephele tasks which read data from a local or distributed file system. As every job input vertex A JobFileInputVertex must not have any further input.

      // create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);


      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));


      o1.setVertexToShareInstancesWith(i1);
      i1.setVertexToShareInstancesWith(t1);


      // connect vertices
      i1.connectTo(t1);
      t1.connectTo(o1);


      LibraryCacheManager.register(jobID, new String[0]);


      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);

View Full Code Here

      // create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);


      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      LibraryCacheManager.register(jobID, new String[0]);


      // now convert job graph to execution graph

View Full Code Here

      // create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(2);
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(2);


      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);
      t1.setNumberOfSubtasks(2);
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask1Input1Output.class);
      t2.setNumberOfSubtasks(2);
      final JobTaskVertex t3 = new JobTaskVertex("Task 3", jg);
      t3.setTaskClass(ForwardTask2Inputs1Output.class);
      t3.setNumberOfSubtasks(2);


      
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(2);
      i1.setVertexToShareInstancesWith(t1);
      t1.setVertexToShareInstancesWith(t3);
      i2.setVertexToShareInstancesWith(t2);
      t2.setVertexToShareInstancesWith(t3);
      t3.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(t1);
      i2.connectTo(t2);
      t1.connectTo(t3);
      t2.connectTo(t3);
      t3.connectTo(o1);


      LibraryCacheManager.register(jobID, new String[0]);

View Full Code Here

      // create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(4);
      i1.setNumberOfSubtasksPerInstance(2);
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(4);
      i2.setNumberOfSubtasksPerInstance(2);
      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);
      t1.setNumberOfSubtasks(4);
      t1.setNumberOfSubtasksPerInstance(2);
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask1Input1Output.class);
      t2.setNumberOfSubtasks(4);
      t2.setNumberOfSubtasksPerInstance(2);
      final JobTaskVertex t3 = new JobTaskVertex("Task 3", jg);
      t3.setTaskClass(ForwardTask2Inputs1Output.class);
      t3.setNumberOfSubtasks(8);
      t3.setNumberOfSubtasksPerInstance(4);
      final JobTaskVertex t4 = new JobTaskVertex("Task 4", jg);
      t4.setTaskClass(ForwardTask1Input2Outputs.class);
      t4.setNumberOfSubtasks(8);
      t4.setNumberOfSubtasksPerInstance(4);
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile1.toURI()));
      o1.setNumberOfSubtasks(4);
      o1.setNumberOfSubtasksPerInstance(2);
      final JobFileOutputVertex o2 = new JobFileOutputVertex("Output 2", jg);
      o2.setFileOutputClass(FileLineWriter.class);
      o2.setFilePath(new Path(outputFile2.toURI()));
      o2.setNumberOfSubtasks(4);
      o2.setNumberOfSubtasksPerInstance(2);
      o1.setVertexToShareInstancesWith(o2);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i2.connectTo(t2, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      t1.connectTo(t3, ChannelType.NETWORK);
      t2.connectTo(t3, ChannelType.NETWORK);
      t3.connectTo(t4, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      t4.connectTo(o1, ChannelType.NETWORK);
      t4.connectTo(o2, ChannelType.NETWORK);

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

eu.stratosphere.core.fs.FileStatus

eu.stratosphere.core.fs.FileSystem

eu.stratosphere.core.fs.Path

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

eu.stratosphere.nephele.jobmanager.JobManagerITCase

java.io.IOException

eu.stratosphere.configuration.IllegalConfigurationException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.