Examples of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

eu.stratosphere.nephele.jobgraph.JobFileInputVertex
A JobFileInputVertex is a specific subtype of a {@link AbstractJobInputVertex} and is designedfor Nephele tasks which read data from a local or distributed file system. As every job input vertex A JobFileInputVertex must not have any further input.

      // create job graph
      final JobGraph jg = new JobGraph("Self Cross Test Job");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex input = new JobFileInputVertex(inputTaskName, jg);
      input.setFileInputClass(SelfCrossInputTask.class);
      input.setFilePath(new Path(inputFile1.toURI()));
      input.setNumberOfSubtasks(degreeOfParallelism);


      // cross vertex
      final JobTaskVertex cross = new JobTaskVertex(crossTaskName, jg);
      cross.setTaskClass(SelfCrossForwardTask.class);
      cross.setNumberOfSubtasks(degreeOfParallelism);


      // output vertex
      final JobFileOutputVertex output = new JobFileOutputVertex(outputTaskName, jg);
      output.setFileOutputClass(FileLineWriter.class);
      output.setFilePath(new Path(outputFile1.toURI()));
      output.setNumberOfSubtasks(degreeOfParallelism);


      // connect vertices
      input.connectTo(cross, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);
      input.connectTo(cross, ChannelType.NETWORK, 1, 1,
        DistributionPattern.BIPARTITE);
      cross.connectTo(output, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);


      LibraryCacheManager.register(jobID, new String[0]);

View Full Code Here

      // create job graph
      final JobGraph jg = new JobGraph("Instance Sharing Test Job");
      jobID = jg.getJobID();


      // input vertex
      final JobFileInputVertex input1 = new JobFileInputVertex("Input 1", jg);
      input1.setFileInputClass(FileLineReader.class);
      input1.setFilePath(new Path(inputFile1.toURI()));
      input1.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 1
      final JobTaskVertex forward1 = new JobTaskVertex("Forward 1", jg);
      forward1.setTaskClass(ForwardTask1Input1Output.class);
      forward1.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 2
      final JobTaskVertex forward2 = new JobTaskVertex("Forward 2", jg);
      forward2.setTaskClass(ForwardTask1Input1Output.class);
      forward2.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 3
      final JobTaskVertex forward3 = new JobTaskVertex("Forward 3", jg);
      forward3.setTaskClass(ForwardTask1Input1Output.class);
      forward3.setNumberOfSubtasks(degreeOfParallelism);


      // output vertex
      final JobFileOutputVertex output1 = new JobFileOutputVertex("Output 1", jg);
      output1.setFileOutputClass(FileLineWriter.class);
      output1.setFilePath(new Path(outputFile1.toURI()));
      output1.setNumberOfSubtasks(degreeOfParallelism);


      // connect vertices
      input1.connectTo(forward1, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
      forward1.connectTo(forward2, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
      forward2.connectTo(forward3, ChannelType.NETWORK,
        DistributionPattern.POINTWISE);
      forward3.connectTo(output1, ChannelType.IN_MEMORY);


      // setup instance sharing
      input1.setVertexToShareInstancesWith(forward1);
      forward1.setVertexToShareInstancesWith(forward2);
      forward2.setVertexToShareInstancesWith(forward3);
      forward3.setVertexToShareInstancesWith(output1);


      LibraryCacheManager.register(jobID, new String[0]);

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(new File(testDirectory).toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(ExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + exceptionClassName + ".jar")
        .toURI()));

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(RuntimeExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + runtimeExceptionClassName
        + ".jar").toURI()));

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph 1");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job Graph for Double Connection Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input with two Outputs", jg);
      i1.setFileInputClass(DoubleSourceTask.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with two Inputs", jg);
      t1.setTaskClass(DoubleTargetTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i1.connectTo(t1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex(jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex(jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Union job " + limit);


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));


      // union task
      final JobTaskVertex u1 = new JobTaskVertex("Union", jg);
      u1.setTaskClass(UnionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      u1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(u1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i2.connectTo(u1, ChannelType.IN_MEMORY);
      u1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));

View Full Code Here


      // Create job graph
      final JobGraph jg = new JobGraph("Job with large DoP (" + numberOfSubtasks + ")");


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(numberOfSubtasks);
      i1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(numberOfSubtasks);
      i2.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // union task
      final JobTaskVertex f1 = new JobTaskVertex("Forward 1", jg);
      f1.setTaskClass(DoubleTargetTask.class);
      f1.setNumberOfSubtasks(numberOfSubtasks);
      f1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(numberOfSubtasks);
      o1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      f1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      i2.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      f1.connectTo(o1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobFileInputVertex

eu.stratosphere.core.fs.FileStatus

eu.stratosphere.core.fs.FileSystem

eu.stratosphere.core.fs.Path

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

eu.stratosphere.nephele.jobmanager.JobManagerITCase

java.io.IOException

eu.stratosphere.configuration.IllegalConfigurationException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.