Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobFileOutputVertex

566
567
568
569
570
571
572
573
574
575
576
577
      // assign input splits
      groupVertex.setInputSplits(inputSplits);
    }
    // TODO: This is a quick workaround, problem can be solved in a more generic way
    if (jobVertex instanceof JobFileOutputVertex) {
      final JobFileOutputVertex jbov = (JobFileOutputVertex) jobVertex;
      jobVertex.getConfiguration().setString("outputPath", jbov.getFilePath().toString());
    }

    // Add group vertex to initial execution stage
    initialExecutionStage.addStageMember(groupVertex);
View Full Code Here
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);

      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));

      o1.setVertexToShareInstancesWith(i1);
      i1.setVertexToShareInstancesWith(t1);

      // connect vertices
      i1.connectTo(t1);
      t1.connectTo(o1);
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
482
483
      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);

      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);
View Full Code Here
597
598
599
600
601
602
603
604
605
606
607
608
609
610
      t3.setTaskClass(ForwardTask2Inputs1Output.class);
      t3.setNumberOfSubtasks(2);

     
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(2);
      i1.setVertexToShareInstancesWith(t1);
      t1.setVertexToShareInstancesWith(t3);
      i2.setVertexToShareInstancesWith(t2);
      t2.setVertexToShareInstancesWith(t3);
      t3.setVertexToShareInstancesWith(o1);
View Full Code Here
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
      final JobTaskVertex t4 = new JobTaskVertex("Task 4", jg);
      t4.setTaskClass(ForwardTask1Input2Outputs.class);
      t4.setNumberOfSubtasks(8);
      t4.setNumberOfSubtasksPerInstance(4);
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile1.toURI()));
      o1.setNumberOfSubtasks(4);
      o1.setNumberOfSubtasksPerInstance(2);
      final JobFileOutputVertex o2 = new JobFileOutputVertex("Output 2", jg);
      o2.setFileOutputClass(FileLineWriter.class);
      o2.setFilePath(new Path(outputFile2.toURI()));
      o2.setNumberOfSubtasks(4);
      o2.setNumberOfSubtasksPerInstance(2);
      o1.setVertexToShareInstancesWith(o2);

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i2.connectTo(t2, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
View Full Code Here
965
966
967
968
969
970
971
972
973
974
975
976
977
978
      final JobTaskVertex cross = new JobTaskVertex(crossTaskName, jg);
      cross.setTaskClass(SelfCrossForwardTask.class);
      cross.setNumberOfSubtasks(degreeOfParallelism);

      // output vertex
      final JobFileOutputVertex output = new JobFileOutputVertex(outputTaskName, jg);
      output.setFileOutputClass(FileLineWriter.class);
      output.setFilePath(new Path(outputFile1.toURI()));
      output.setNumberOfSubtasks(degreeOfParallelism);

      // connect vertices
      input.connectTo(cross, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);
      input.connectTo(cross, ChannelType.NETWORK, 1, 1,
View Full Code Here
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
      final JobTaskVertex forward3 = new JobTaskVertex("Forward 3", jg);
      forward3.setTaskClass(ForwardTask1Input1Output.class);
      forward3.setNumberOfSubtasks(degreeOfParallelism);

      // output vertex
      final JobFileOutputVertex output1 = new JobFileOutputVertex("Output 1", jg);
      output1.setFileOutputClass(FileLineWriter.class);
      output1.setFilePath(new Path(outputFile1.toURI()));
      output1.setNumberOfSubtasks(degreeOfParallelism);

      // connect vertices
      input1.connectTo(forward1, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
      forward1.connectTo(forward2, ChannelType.IN_MEMORY,
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
View Full Code Here
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(ExceptionTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);
View Full Code Here
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(RuntimeExceptionTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);
View Full Code Here
TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobFileOutputVertex

  • eu.stratosphere.core.fs.FileStatus
  • eu.stratosphere.core.fs.FileSystem
  • eu.stratosphere.core.fs.Path
  • eu.stratosphere.nephele.executiongraph.ExecutionGraph
  • eu.stratosphere.nephele.executiongraph.ExecutionGraphTest
  • eu.stratosphere.nephele.jobmanager.JobManagerITCase
  • eu.stratosphere.configuration.IllegalConfigurationException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.