Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobTracker

165
166
167
168
169
170
171
172
173
174
175
176
        return myJob;
      }
    });

    // Make the job go into RUNNING state by forceful initialization.
    JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
    JobInProgress jip =
        jt.getJob(org.apache.hadoop.mapred.JobID.downgrade(job.getJobID()));
    jt.initJob(jip);

    return job;
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    return new Token<DelegationTokenIdentifier>(dtId, dtSecretManager);
  }

  @Test
  public void testDelegationToken() throws Exception {
    final JobTracker jt = cluster.getJobTrackerRunner().getJobTracker();
    final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
    config.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
        "kerberos");
    // Set configuration again so that job tracker finds security enabled
    UserGroupInformation.setConfiguration(config);
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws Exception {
        try {
          Token<DelegationTokenIdentifier> token = jt
              .getDelegationToken(new Text(ugi.getShortUserName()));
          jt.renewDelegationToken(token);
          jt.cancelDelegationToken(token);
        } catch (IOException e) {
          e.printStackTrace();
          throw e;
        }
        return null;
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
    });
  }
 
  @Test
  public void testGetDelegationTokenWithoutKerberos() throws Exception {
    final JobTracker jt = cluster.getJobTrackerRunner().getJobTracker();
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
    config.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
        "kerberos");
    // Set configuration again so that job tracker finds security enabled
    UserGroupInformation.setConfiguration(config);
    Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws Exception {
        try {
          Token<DelegationTokenIdentifier> token = jt
              .getDelegationToken(new Text("arenewer"));
          Assert.assertTrue(token != null);
          Assert
              .fail("Delegation token should not be issued without Kerberos authentication");
        } catch (IOException e) {
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
    });
  }

  @Test
  public void testRenewDelegationTokenWithoutKerberos() throws Exception {
    final JobTracker jt = cluster.getJobTrackerRunner().getJobTracker();
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
    config.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
        "kerberos");
    // Set configuration again so that job tracker finds security enabled
    UserGroupInformation.setConfiguration(config);
    Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
    final Token<DelegationTokenIdentifier> token = generateDelegationToken(
        "owner", ugi.getShortUserName());
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
      public Object run() throws Exception {
        try {
          jt.renewDelegationToken(token);
          Assert
              .fail("Delegation token should not be renewed without Kerberos authentication");
        } catch (IOException e) {
          // success
        }
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
    }

    @Override
    public void shutdown() throws IOException {
      MiniMRCluster.JobTrackerRunner runner = mr.getJobTrackerRunner();
      JobTracker tracker = runner.getJobTracker();
      if (tracker != null) {
        for (JobInProgress running : tracker.getRunningJobs()) {
          try {
            running.kill();
          } catch (Exception e) {
            // ignore
          }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
        return myJob;
      }
    });

    // Make the job go into RUNNING state by forceful initialization.
    JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
    JobInProgress jip =
        jt.getJob(org.apache.hadoop.mapred.JobID.downgrade(job.getJobID()));
    jt.initJob(jip);

    return job;
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
            NameNode.format(conf);
            final String[] emptyArgs = {};
            NameNode.createNameNode(emptyArgs, conf);
            DataNode.createDataNode(emptyArgs, conf);
            final JobTracker jobTracker = JobTracker.startTracker(new JobConf(conf));
            new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        jobTracker.offerService();
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }).start();
View Full Code Here
30
31
32
33
34
35
36
37
38
        for (Enumeration e = properties.keys(); e.hasMoreElements();) {
            Object key = e.nextElement();
            Object val = properties.get(key);
            conf.set( key.toString(), val.toString() );
        }
        JobTracker jobTracker = JobTracker.startTracker(conf);
        jobTracker.offerService();
        return jobTracker;
    }
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
     */
    private static Thread getJobTrackerThread() {
        Thread jobTrackerThread = new Thread(new Runnable() {

            public void run() {
                JobTracker jobTracker = null;

                while (true) {
                    try {
                        jobTracker = JobTracker.startTracker(new CassandraJobConf());
                        logger.info("Hadoop Job Tracker Started...");
                        jobTrackerStarted.countDown();
                        jobTracker.offerService();

                    } catch (Throwable t) {
                        if (t instanceof InterruptedException) {
                            try {
                                jobTracker.stopTracker();
                                logger.info("Job Tracker shutdown property");
                            } catch (Exception e) {
                                logger.error("An Error occured when stopping Job tracker");
                            }
                        }

                        // on OOM shut down the tracker
                        if (t instanceof OutOfMemoryError || t.getCause() instanceof OutOfMemoryError) {
                            try {
                                jobTracker.stopTracker();
                            } catch (IOException e) {

                            }
                            logger.warn("Error starting job tracker", t);
                            break;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.JobTracker

  • io.fabric8.hadoop.mapred.JobTrackerFactory
  • org.apache.cassandra.hadoop.trackers.TrackerInitializer
  • org.apache.falcon.listener.HadoopStartupListener
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileStatus
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.FSDataOutputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.PathFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.