Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.Path

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

  @Override
  public void invoke() throws Exception {

    Path outputPath = getFileOutputPath();

    FileSystem fs = FileSystem.get(outputPath.toUri());
    if (fs.exists(outputPath)) {
      FileStatus status = fs.getFileStatus(outputPath);

      if (status.isDir()) {
        outputPath = new Path(outputPath.toUri().toString() + "/file_" + getIndexInSubtaskGroup() + ".txt");
      }
    }

    final FSDataOutputStream outputStream = fs.create(outputPath, true);
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));

      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);

      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));

      o1.setVertexToShareInstancesWith(i1);
      i1.setVertexToShareInstancesWith(t1);

      // connect vertices
View Full Code Here
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));

      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);

      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(new File(ServerTestUtils.getRandomFilename()).toURI()));

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);
View Full Code Here
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(2);
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(2);

      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);
      t1.setNumberOfSubtasks(2);
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask1Input1Output.class);
      t2.setNumberOfSubtasks(2);
      final JobTaskVertex t3 = new JobTaskVertex("Task 3", jg);
      t3.setTaskClass(ForwardTask2Inputs1Output.class);
      t3.setNumberOfSubtasks(2);

     
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(2);
      i1.setVertexToShareInstancesWith(t1);
      t1.setVertexToShareInstancesWith(t3);
      i2.setVertexToShareInstancesWith(t2);
      t2.setVertexToShareInstancesWith(t3);
View Full Code Here
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(4);
      i1.setNumberOfSubtasksPerInstance(2);
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(4);
      i2.setNumberOfSubtasksPerInstance(2);
      // task vertex
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask1Input1Output.class);
      t1.setNumberOfSubtasks(4);
      t1.setNumberOfSubtasksPerInstance(2);
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask1Input1Output.class);
      t2.setNumberOfSubtasks(4);
      t2.setNumberOfSubtasksPerInstance(2);
      final JobTaskVertex t3 = new JobTaskVertex("Task 3", jg);
      t3.setTaskClass(ForwardTask2Inputs1Output.class);
      t3.setNumberOfSubtasks(8);
      t3.setNumberOfSubtasksPerInstance(4);
      final JobTaskVertex t4 = new JobTaskVertex("Task 4", jg);
      t4.setTaskClass(ForwardTask1Input2Outputs.class);
      t4.setNumberOfSubtasks(8);
      t4.setNumberOfSubtasksPerInstance(4);
      // output vertex
      final JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile1.toURI()));
      o1.setNumberOfSubtasks(4);
      o1.setNumberOfSubtasksPerInstance(2);
      final JobFileOutputVertex o2 = new JobFileOutputVertex("Output 2", jg);
      o2.setFileOutputClass(FileLineWriter.class);
      o2.setFilePath(new Path(outputFile2.toURI()));
      o2.setNumberOfSubtasks(4);
      o2.setNumberOfSubtasksPerInstance(2);
      o1.setVertexToShareInstancesWith(o2);

      // connect vertices
View Full Code Here
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex input = new JobFileInputVertex(inputTaskName, jg);
      input.setFileInputClass(SelfCrossInputTask.class);
      input.setFilePath(new Path(inputFile1.toURI()));
      input.setNumberOfSubtasks(degreeOfParallelism);

      // cross vertex
      final JobTaskVertex cross = new JobTaskVertex(crossTaskName, jg);
      cross.setTaskClass(SelfCrossForwardTask.class);
      cross.setNumberOfSubtasks(degreeOfParallelism);

      // output vertex
      final JobFileOutputVertex output = new JobFileOutputVertex(outputTaskName, jg);
      output.setFileOutputClass(FileLineWriter.class);
      output.setFilePath(new Path(outputFile1.toURI()));
      output.setNumberOfSubtasks(degreeOfParallelism);

      // connect vertices
      input.connectTo(cross, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);
View Full Code Here
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
      jobID = jg.getJobID();

      // input vertex
      final JobFileInputVertex input1 = new JobFileInputVertex("Input 1", jg);
      input1.setFileInputClass(FileLineReader.class);
      input1.setFilePath(new Path(inputFile1.toURI()));
      input1.setNumberOfSubtasks(degreeOfParallelism);

      // forward vertex 1
      final JobTaskVertex forward1 = new JobTaskVertex("Forward 1", jg);
      forward1.setTaskClass(ForwardTask1Input1Output.class);
      forward1.setNumberOfSubtasks(degreeOfParallelism);

      // forward vertex 2
      final JobTaskVertex forward2 = new JobTaskVertex("Forward 2", jg);
      forward2.setTaskClass(ForwardTask1Input1Output.class);
      forward2.setNumberOfSubtasks(degreeOfParallelism);

      // forward vertex 3
      final JobTaskVertex forward3 = new JobTaskVertex("Forward 3", jg);
      forward3.setTaskClass(ForwardTask1Input1Output.class);
      forward3.setNumberOfSubtasks(degreeOfParallelism);

      // output vertex
      final JobFileOutputVertex output1 = new JobFileOutputVertex("Output 1", jg);
      output1.setFileOutputClass(FileLineWriter.class);
      output1.setFilePath(new Path(outputFile1.toURI()));
      output1.setNumberOfSubtasks(degreeOfParallelism);

      // connect vertices
      input1.connectTo(forward1, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
      final JobGraph jg = new JobGraph("Job Graph 1");

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(new File(testDirectory).toURI()));

      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);

      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }

      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + forwardClassName + ".jar").toURI()));

      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));

      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(ExceptionTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);

      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + exceptionClassName + ".jar")
        .toURI()));

      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
     
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
      final JobGraph jg = new JobGraph("Job Graph for Exception Test");

      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));

      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(RuntimeExceptionTask.class);

      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));

      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);

      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);

      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + runtimeExceptionClassName
        + ".jar").toURI()));

      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
     
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.Path

  • eu.stratosphere.api.avro.testjar.AvroExternalJarProgram
  • eu.stratosphere.api.common.cache.DistributedCache
  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread
  • eu.stratosphere.api.common.io.FileOutputFormatTest
  • eu.stratosphere.api.common.io.FormatUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.