Package org.apache.hadoop.tools.rumen

Examples of org.apache.hadoop.tools.rumen.JobStory

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
              }
            }

            while (!loadStatus.overloaded()) {
              try {
                final JobStory job = getNextJobFiltered();
                if (null == job) {
                  return;
                }
               
                submitter.add(
                  jobCreator.createGridmixJob(
                    conf, 0L, job, scratch, userResolver.getTargetUgi(
                      UserGroupInformation.createRemoteUser(
                        job.getUser())), sequence.getAndIncrement()));
                // TODO: We need to take care of scenario when one map/reduce
                // takes more than 1 slot.
                loadStatus.mapSlotsBackfill -=
                  calcEffectiveIncompleteMapTasks(
                    loadStatus.mapSlotCapacity, job.getNumberMaps(), 0.0f);
                loadStatus.reduceSlotsBackfill -=
                  calcEffectiveIncompleteReduceTasks(
                    loadStatus.reduceSlotCapacity, job.getNumberReduces(),
                    0.0f);
                --loadStatus.numJobsBackfill;
              } catch (IOException e) {
                LOG.error("Error while submitting the job ", e);
                error = e;
View Full Code Here
454
455
456
457
458
459
460
461
462
463
464
465
466
   */
  @Test (timeout=30000)
  public void testCompareGridmixJob() throws Exception {
    Configuration conf = new Configuration();
    Path outRoot = new Path("target");
    JobStory jobDesc = mock(JobStory.class);
    when(jobDesc.getName()).thenReturn("JobName");
    when(jobDesc.getJobConf()).thenReturn(new JobConf(conf));
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    GridmixJob j1 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
    GridmixJob j2 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 0);
    GridmixJob j3 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
    GridmixJob j4 = new LoadJob(conf, 1000L, jobDesc, outRoot, ugi, 1);
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
  private void testRandomLocation(int locations, int njobs, UserGroupInformation ugi) throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations);
    DebugJobProducer jobProducer = new DebugJobProducer(njobs, conf);
    JobConf jconf = GridmixTestUtils.mrCluster.createJobConf(new JobConf(conf));
    JobStory story;
    int seq=1;
    while ((story = jobProducer.getNextJob()) != null) {
      GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0,
          story, new Path("ignored"), ugi, seq++);
      gridmixJob.buildSplits(null);
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
    conf.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true);
    conf.set("mapreduce.job.hdfs-servers", "");
    DebugJobProducer jobProducer = new DebugJobProducer(5, conf);
    JobConf jconf = GridmixTestUtils.mrCluster.createJobConf(new JobConf(conf));
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    JobStory story;
    int seq = 1;
    while ((story = jobProducer.getNextJob()) != null) {
      GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0,
          story, new Path("ignored"), ugi, seq++);
      gridmixJob.buildSplits(null);
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
              conf.get(MRJobConfig.QUEUE_NAME),
              sub.get(conf.get(Gridmix.ORIGINAL_JOB_ID)).getQueueName());
        }

        final String originalJobId = conf.get(Gridmix.ORIGINAL_JOB_ID);
        final JobStory spec = sub.get(originalJobId);
        assertNotNull("No spec for " + jobName, spec);
        assertNotNull("No counters for " + jobName, job.getCounters());
        final String originalJobName = spec.getName();
        System.out.println("originalJobName=" + originalJobName
            + ";GridmixJobName=" + jobName + ";originalJobID=" + originalJobId);
        assertTrue("Original job name is wrong.", originalJobName.equals(
            conf.get(Gridmix.ORIGINAL_JOB_NAME)));

        // Gridmix job seqNum contains 6 digits
        int seqNumLength = 6;
        String jobSeqNum = new DecimalFormat("000000").format(
            conf.getInt(GridmixJob.GRIDMIX_JOB_SEQ, -1));
        // Original job name is of the format MOCKJOB<6 digit sequence number>
        // because MockJob jobNames are of this format.
        assertTrue(originalJobName.substring(
            originalJobName.length() - seqNumLength).equals(jobSeqNum));

        assertTrue("Gridmix job name is not in the expected format.",
            jobName.equals(
            GridmixJob.JOB_NAME_PREFIX + jobSeqNum));

        final FileStatus stat =
          GridmixTestUtils.dfs.getFileStatus(
            new Path(out, "" + Integer.valueOf(jobSeqNum)));
        assertEquals("Wrong owner for " + jobName, spec.getUser(),
                     stat.getOwner());

        final int nMaps = spec.getNumberMaps();
        final int nReds = spec.getNumberReduces();

        // TODO Blocked by MAPREDUCE-118
        if (true) return;
        // TODO
        System.out.println(jobName + ": " + nMaps + "/" + nReds);
View Full Code Here
266
267
268
269
270
271
272
273
274
275
276
277
278
279
   * @throws IOException
   */
  private int buildDistCacheFilesList(JobStoryProducer jsp) throws IOException {
    // Read all the jobs from the trace file and build the list of unique
    // distributed cache files.
    JobStory jobStory;
    while ((jobStory = jsp.getNextJob()) != null) {
      if (jobStory.getOutcome() == Pre21JobHistoryConstants.Values.SUCCESS &&
         jobStory.getSubmissionTime() >= 0) {
        updateHDFSDistCacheFilesList(jobStory);
      }
    }
    jsp.close();

View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    DebugJobProducer jobProducer = new DebugJobProducer(5, configuration);
    configuration.setBoolean(SleepJob.SLEEPJOB_MAPTASK_ONLY, true);

    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    JobStory story;
    int seq = 1;
    while ((story = jobProducer.getNextJob()) != null) {
      GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(configuration, 0,
              story, new Path("ignored"), ugi, seq++);
      gridmixJob.buildSplits(null);
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
    DebugJobProducer jobProducer = new DebugJobProducer(njobs, configuration);
    Configuration jconf = GridmixTestUtils.mrvl.getConfig();
    jconf.setInt(JobCreator.SLEEPJOB_RANDOM_LOCATIONS, locations);

    JobStory story;
    int seq = 1;
    while ((story = jobProducer.getNextJob()) != null) {
      GridmixJob gridmixJob = JobCreator.SLEEPJOB.createGridmixJob(jconf, 0,
              story, new Path("ignored"), ugi, seq++);
      gridmixJob.buildSplits(null);
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
              if (LOG.isDebugEnabled()) {
                LOG.debug("Cluster underloaded in run! Stressing...");
              }
              try {
                //TODO This in-line read can block submission for large jobs.
                final JobStory job = getNextJobFiltered();
                if (null == job) {
                  return;
                }
                if (LOG.isDebugEnabled()) {
                  LOG.debug("Job Selected: " + job.getJobID());
                }
                submitter.add(
                  jobCreator.createGridmixJob(
                    conf, 0L, job, scratch,
                    userResolver.getTargetUgi(
                      UserGroupInformation.createRemoteUser(job.getUser())),
                    sequence.getAndIncrement()));
                // TODO: We need to take care of scenario when one map/reduce
                // takes more than 1 slot.
               
                // Lock the loadjob as we are making updates
                int incompleteMapTasks = (int) calcEffectiveIncompleteMapTasks(
                                                 loadStatus.getMapCapacity(),
                                                 job.getNumberMaps(), 0.0f);
                loadStatus.decrementMapLoad(incompleteMapTasks);
               
                int incompleteReduceTasks =
                  (int) calcEffectiveIncompleteReduceTasks(
                          loadStatus.getReduceCapacity(),
                          job.getNumberReduces(), 0.0f);
                loadStatus.decrementReduceLoad(incompleteReduceTasks);
                 
                loadStatus.decrementJobLoad(1);
              } catch (IOException e) {
                LOG.error("Error while submitting the job ", e);
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
              (GENDATA << 20) > generated.getLength() - GENSLOP);
          FileStatus[] outstat = dfs.listStatus(out);
          assertEquals("Mismatched job count", NJOBS, outstat.length);
          continue;
        }
        final JobStory spec =
          sub.get(job.getJobName().replace("GRIDMIX", "MOCKJOB"));
        assertNotNull("No spec for " + job.getJobName(), spec);
        assertNotNull("No counters for " + job.getJobName(), job.getCounters());

        final int nMaps = spec.getNumberMaps();
        final int nReds = spec.getNumberReduces();

        System.out.println(jobname + ": " + nMaps + "/" + nReds);
        final TaskReport[] mReports = job.getTaskReports(TaskType.MAP);
        assertEquals("Mismatched map count", nMaps, mReports.length);
        check(TaskType.MAP, job, spec, mReports,
View Full Code Here
TOP

Related Classes of org.apache.hadoop.tools.rumen.JobStory

  • org.apache.hadoop.mapred.gridmix.CommonJobTest$TestMonitor
  • org.apache.hadoop.mapred.gridmix.DistributedCacheEmulator
  • org.apache.hadoop.mapred.gridmix.GridmixJob
  • org.apache.hadoop.mapred.gridmix.JobFactory
  • org.apache.hadoop.mapred.gridmix.JobFactory$ReaderThread
  • org.apache.hadoop.mapred.gridmix.LoadJob
  • org.apache.hadoop.mapred.gridmix.ReplayJobFactory$ReplayReaderThread
  • org.apache.hadoop.mapred.gridmix.SerialJobFactory$SerialReaderThread
  • org.apache.hadoop.mapred.gridmix.StressJobFactory$StressReaderThread
  • org.apache.hadoop.mapred.gridmix.TestGridMixClasses
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.