Package com.cloudera.iterativereduce.yarn.avro.generated

Examples of com.cloudera.iterativereduce.yarn.avro.generated.StartupConfiguration

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
                    + "unknown.").build();
      }
     
      // TODO: can a worker "start" more than once?
     
      StartupConfiguration workerConf = workers.get(workerId);
      Utils.mergeConfigs(appConf, workerConf);

      LOG.debug("Got a startup call, workerId="
          + Utils.getWorkerId(workerId) + ", responded with"
          + ", batchSize=" + workerConf.getBatchSize() + ", iterations="
          + workerConf.getIterations() + ", fileSplit=[" + workerConf.getSplit().getPath()
          + ", " + workerConf.getSplit().getOffset() + "]");

      workersState.put(workerId, WorkerState.STARTED);

      return workerConf;
    }
View Full Code Here
249
250
251
252
253
254
255
256
257
258
259
     
      convertedToMetronomeSplit.length = hadoopFileSplit.getLength();
      convertedToMetronomeSplit.offset = hadoopFileSplit.getStart();
      convertedToMetronomeSplit.path = hadoopFileSplit.getPath().toString();
     
        StartupConfiguration config = StartupConfiguration.newBuilder()
              .setBatchSize(batchSize).setIterations(iterationCount)
              .setOther(appConfig).setSplit( convertedToMetronomeSplit ).build();
             
        String wid = "worker-" + workerId;
        ConfigurationTuple tuple = new ConfigurationTuple( split.getLocations()[ 0 ], wid, config );
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
      for (BlockLocation b : bl) {
        FileSplit split = FileSplit.newBuilder().setPath(p.toString())
            .setOffset(b.getOffset()).setLength(b.getLength()).build();

        StartupConfiguration config = StartupConfiguration.newBuilder()
            .setBatchSize(batchSize).setIterations(iterationCount)
            .setOther(appConfig).setSplit(split).build();

        String wid = "worker-" + workerId;
        ConfigurationTuple tuple = new ConfigurationTuple(b.getHosts()[0], wid,
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
  public void setUpMaster() throws Exception {
    FileSplit split = FileSplit.newBuilder()
        .setPath("testData/testWorkerService.txt").setOffset(0).setLength(200)
        .build();

    StartupConfiguration conf = StartupConfiguration.newBuilder()
        .setSplit(split).setBatchSize(2).setIterations(1).setOther(null)
        .build();

    HashMap<WorkerId, StartupConfiguration> workers = new HashMap<WorkerId, StartupConfiguration>();
    workers.put(Utils.createWorkerId("worker1"), conf);
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  public void setUpMaster() throws Exception {
    FileSplit split = FileSplit.newBuilder()
        .setPath("testData/testWorkerService.txt").setOffset(0).setLength(200)
        .build();

    StartupConfiguration conf = StartupConfiguration.newBuilder()
        .setSplit(split).setBatchSize(200).setIterations(1).setOther(null)
        .build();

    HashMap<WorkerId, StartupConfiguration> workers = new HashMap<WorkerId, StartupConfiguration>();
    workers.put(Utils.createWorkerId("worker1"), conf);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
  public void setUpMaster() throws Exception {
    FileSplit split = FileSplit.newBuilder()
        .setPath("testData/testWorkerService.txt").setOffset(0).setLength(200)
        .build();

    StartupConfiguration conf = StartupConfiguration.newBuilder()
        .setSplit(split).setBatchSize(200).setIterations(1).setOther(null)
        .build();

    HashMap<WorkerId, StartupConfiguration> workers = new HashMap<WorkerId, StartupConfiguration>();
    workers.put(Utils.createWorkerId("worker1"), conf);
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
        .setPath("/foo/bar")
        .setOffset(100)
        .setLength(200)
        .build();
   
    StartupConfiguration conf = StartupConfiguration.newBuilder()
        .setSplit(split)
        .setBatchSize(2)
        .setIterations(1)
        .setOther(null)
        .build();
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
  @Before
  public void setUp() throws Exception {
    FileSplit split = FileSplit.newBuilder().setPath("/foo/bar").setOffset(100)
        .setLength(200).build();

    StartupConfiguration conf = StartupConfiguration.newBuilder()
        .setSplit(split).setBatchSize(2).setIterations(1).setOther(null)
        .build();

    HashMap<WorkerId, StartupConfiguration> workers = new HashMap<WorkerId, StartupConfiguration>();
    workers.put(Utils.createWorkerId("worker1"), conf);
View Full Code Here
TOP

Related Classes of com.cloudera.iterativereduce.yarn.avro.generated.StartupConfiguration

  • com.cloudera.iterativereduce.yarn.appmaster.ApplicationMaster
  • com.cloudera.iterativereduce.yarn.appmaster.ApplicationMasterService
  • com.cloudera.iterativereduce.yarn.appmaster.ErroneousMasterShutdownTest
  • com.cloudera.iterativereduce.yarn.appmaster.MasterServiceRPCTest
  • com.cloudera.iterativereduce.yarn.appworker.ApplicationWorkerTest
  • com.cloudera.iterativereduce.yarn.appworker.MultipleWorkerServicesTest
  • com.cloudera.iterativereduce.yarn.appworker.SingleWorkerServiceTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.