Package eu.stratosphere.pact.runtime.task.util

Examples of eu.stratosphere.pact.runtime.task.util.TaskConfig

345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
    return fakeTailOutput;
  }

  private static JobOutputVertex createSync(JobGraph jobGraph, int numSubTasks, int maxIterations) {
    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      LongSumAggregator.class);
    syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      WorksetEmptyConvergenceCriterion.class);

    return sync;
  }
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
248
  private static JobInputVertex createModelsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
    CsvInputFormat modelsInFormat = new CsvInputFormat(' ', LongValue.class, LongValue.class, LongValue.class, LongValue.class);
    JobInputVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, pointsPath, "Input[Models]", jobGraph, numSubTasks, numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
      taskConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
      taskConfig.setOutputSerializer(serializer);
    }

    return modelsInput;
  }
View Full Code Here
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
  private static JobTaskVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
    JobTaskVertex pointsInput = JobGraphUtils.createTask(RegularPactTask.class, "Map[DotProducts]", jobGraph, numSubTasks, numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());

      taskConfig.setStubWrapper(new UserCodeClassWrapper<DotProducts>(DotProducts.class));
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
      taskConfig.setDriver(CollectorMapDriver.class);
      taskConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);

      taskConfig.addInputToGroup(0);
      taskConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      taskConfig.setInputSerializer(serializer, 0);

      taskConfig.setBroadcastInputName("models", 0);
      taskConfig.addBroadcastInputToGroup(0);
      taskConfig.setBroadcastInputSerializer(serializer, 0);
    }

    return pointsInput;
  }
View Full Code Here
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    JobTaskVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);

    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());

    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);

    // --------------- the tail (solution set join) ---------------
    JobTaskVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph,
      numSubTasks, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);

      tailConfig.setIsWorksetIteration();
      tailConfig.setIsWorksetUpdate();

      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();

      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);

      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);

      // the driver
      tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setDriverPairComparator(pairComparator);
     
      tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }

    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);

    JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);

    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(tail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    return true;
  }

  @Override
  public void prepare() throws Exception{
    final TaskConfig config = this.taskContext.getTaskConfig();
   
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
   
    // set up memory and I/O parameters
    final long availableMemory = config.getMemoryDriver();
    final int numPages = memoryManager.computeNumberOfPages(availableMemory);
   
    // test minimum memory requirements
    final DriverStrategy ls = config.getDriverStrategy();
   
    final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
    final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
   
    // get the key positions and types
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> comparator1 = this.taskContext.getInputComparator(0);
    final TypeComparator<IT2> comparator2 = this.taskContext.getInputComparator(1);
   
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
        this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
      throw new Exception("Missing pair comparator factory for Match driver");
    }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
      this.hashTable = (CompactingHashTable<IT1>) SolutionSetBroker.instance().get(identifier);
    } else {
      throw new RuntimeException("The task context of this driver is no iterative task context.");
    }
   
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
   
    TypeSerializer<IT1> solutionSetSerializer = this.hashTable.getBuildSideSerializer();
    TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
   
    TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    TypeComparator<IT1> solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
   
    solutionSideRecord = solutionSetSerializer.createInstance();
    probeSideRecord = probeSideSerializer.createInstance();
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  // --------------------------------------------------------------------------------------------

  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    this.strategy = config.getDriverStrategy();
   
    if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
      if (!(this.taskContext.getStub() instanceof GenericCombine)) {
        throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GenericCombine.class.getName());
      }
    }
    else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
      throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + config.getDriverStrategy().name());
    }
    this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
    this.input = this.taskContext.getInput(0);
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    return true;
  }

  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final DriverStrategy ls = config.getDriverStrategy();

    final long availableMemory = config.getMemoryDriver();

    final MemoryManager memoryManager = this.taskContext.getMemoryManager();

    final MutableObjectIterator<T> in = this.taskContext.getInput(0);
    this.serializerFactory = this.taskContext.getInputSerializer(0);
View Full Code Here
TOP

Related Classes of eu.stratosphere.pact.runtime.task.util.TaskConfig

  • eu.stratosphere.api.common.distributions.DataDistribution
  • eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator
  • eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator$IterationDescriptor
  • eu.stratosphere.configuration.Configuration
  • eu.stratosphere.pact.runtime.iterative.task.AbstractIterativePactTask
  • eu.stratosphere.pact.runtime.iterative.task.IterationHeadPactTask
  • eu.stratosphere.pact.runtime.iterative.task.IterationSynchronizationSinkTask
  • eu.stratosphere.pact.runtime.task.AbstractCachedBuildSideMatchDriver
  • eu.stratosphere.pact.runtime.task.AllGroupReduceDriver
  • eu.stratosphere.pact.runtime.task.AllReduceDriver
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.