Package org.apache.flink.runtime.operators.util

Examples of org.apache.flink.runtime.operators.util.TaskConfig

117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    }
    else {
      throw new Exception("The task context of this driver is no iterative task context.");
    }
   
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
   
    TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
   
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
   
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
   
    solutionSideRecord = solutionSetSerializer.createInstance();
    probeSideRecord = probeSideSerializer.createInstance();
View Full Code Here
286
287
288
289
290
291
292
293
294
295
296
   */
  private void initInputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);

    try {
      this.format = config.<InputFormat<OT, InputSplit>>getStubWrapper(userCodeClassLoader)
          .getUserCodeObject(InputFormat.class, userCodeClassLoader);

View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
  }


  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final DriverStrategy ls = config.getDriverStrategy();
   
    switch (ls)
    {
    case NESTEDLOOP_BLOCKED_OUTER_FIRST:
      this.blocked = true;
      this.firstIsOuter = true;
      break;
    case NESTEDLOOP_BLOCKED_OUTER_SECOND:
      this.blocked = true;
      this.firstIsOuter = false;
      break;
    case NESTEDLOOP_STREAMED_OUTER_FIRST:
      this.blocked = false;
      this.firstIsOuter = true;
      break;
    case NESTEDLOOP_STREAMED_OUTER_SECOND:
      this.blocked = false;
      this.firstIsOuter = false;
      break;
    default:
      throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
    }
   
    this.memManager = this.taskContext.getMemoryManager();
    final int numPages = this.memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
   
    if (numPages < 2) {
      throw new RuntimeException(  "The Cross task was initialized with too little memory. " +
          "Cross requires at least 2 memory pages.");
    }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
 
  @Override
  public void initializeOnMaster(ClassLoader loader) throws Exception {
    if (inputFormat == null) {
      TaskConfig cfg = new TaskConfig(getConfiguration());
      UserCodeWrapper<InputFormat<?, ?>> wrapper = cfg.<InputFormat<?, ?>>getStubWrapper(loader);
     
      if (wrapper == null) {
        throw new Exception("No input format present in InputFormatVertex's task configuration.");
      }
     
      inputFormat = wrapper.getUserCodeObject(InputFormat.class, loader);
      inputFormat.configure(cfg.getStubParameters());
    }
   
    setInputSplitSource(inputFormat);
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    }
    else {
      throw new Exception("The task context of this driver is no iterative task context.");
    }
   
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
   
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
   
    this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
   
    solutionSideRecord = solutionSetSerializer.createInstance();
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    super.initOutputs();

    // at this time, the outputs to the step function are created
    // add the outputs for the final solution
    this.finalOutputWriters = new ArrayList<BufferWriter>();
    final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
    final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    this.finalOutputCollector = RegularPactTask.getOutputCollector(this, finalOutConfig,
      userCodeClassLoader, this.finalOutputWriters, finalOutConfig.getNumOutputs());

    // sanity check the setup
    final int writersIntoStepFunction = this.eventualOutputs.size();
    final int writersIntoFinalResult = this.finalOutputWriters.size();
    final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
          excludeFromReset(i);
        }
      }
    }
   
    TaskConfig config = getLastTasksConfig();
    isWorksetIteration = config.getIsWorksetIteration();
    isWorksetUpdate = config.getIsWorksetUpdate();
    isSolutionSetUpdate = config.getIsSolutionSetUpdate();

    if (isWorksetUpdate) {
      worksetBackChannel = BlockingBackChannelBroker.instance().getAndRemove(brokerKey());

      if (isWorksetIteration) {
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
 
  @Override
  public void initializeOnMaster(ClassLoader loader) throws Exception {
    if (this.outputFormat == null) {
      TaskConfig cfg = new TaskConfig(getConfiguration());
      UserCodeWrapper<OutputFormat<?>> wrapper = cfg.<OutputFormat<?>>getStubWrapper(loader);
   
      if (wrapper == null) {
        throw new Exception("No output format present in OutputFormatVertex's task configuration.");
      }

      this.outputFormat = wrapper.getUserCodeObject(OutputFormat.class, loader);
      this.outputFormat.configure(cfg.getStubParameters());
    }
   
    if (this.outputFormat instanceof InitializeOnMaster) {
      ((InitializeOnMaster) this.outputFormat).initializeGlobal(getParallelism());
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    this.headEventReader = new MutableRecordReader<IntegerRecord>(this);
  }

  @Override
  public void invoke() throws Exception {
    TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());
   
    // store all aggregators
    this.aggregators = new HashMap<String, Aggregator<?>>();
    for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators(getUserCodeClassLoader())) {
      aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
    }
   
    // store the aggregator convergence criterion
    if (taskConfig.usesConvergenceCriterion()) {
      convergenceCriterion = taskConfig.getConvergenceCriterion(getUserCodeClassLoader());
      convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
      Preconditions.checkNotNull(convergenceAggregatorName);
    }
   
    maxNumberOfIterations = taskConfig.getNumberOfIterations();
   
    // set up the event handler
    int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
    eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
        getEnvironment().getUserClassLoader());
    headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);

    IntegerRecord dummy = new IntegerRecord();
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
      LOG.debug(formatLogString("Start registering input and output."));
    }

    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);

    // now get the operator class which drives the operation
    final Class<? extends PactDriver<S, OT>> driverClass = this.config.getDriver();
    this.driver = InstantiationUtil.instantiate(driverClass, PactDriver.class);
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.operators.util.TaskConfig

  • org.apache.flink.api.common.distributions.DataDistribution
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator$IterationDescriptor
  • org.apache.flink.configuration.Configuration
  • org.apache.flink.core.memory.InputViewDataInputStreamWrapper
  • org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper
  • org.apache.flink.runtime.iterative.task.AbstractIterativePactTask
  • org.apache.flink.runtime.iterative.task.IterationHeadPactTask
  • org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask
  • org.apache.flink.runtime.jobgraph.InputFormatVertex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.