Package org.apache.hadoop.util

Examples of org.apache.hadoop.util.MRAsyncDiskService$DeleteTask

613
614
615
616
617
618
619
620
621
622
623
    }
    // Check local disk, start async disk service, and clean up all
    // local directories.
    checkLocalDirs(this.fConf.getLocalDirs());
    asyncDiskService = new MRAsyncDiskService(fConf);
    asyncDiskService.cleanupAllVolumes();

    // Clear out state tables
    this.tasks.clear();
    this.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
View Full Code Here
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
   
    // start the recovery manager
    recoveryManager = new RecoveryManager();
   
    // start async disk service for asynchronous deletion service
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(jobConf),
        jobConf.getLocalDirs());
   
    while (!Thread.currentThread().isInterrupted()) {
      try {
        // if we haven't contacted the namenode go ahead and do it
View Full Code Here
142
143
144
145
146
147
148
149
150
    // Put a file in one of the volumes to be cleared on startup.
    Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED);
    localFileSystem.mkdirs(delDir);
    localFileSystem.create(new Path(delDir, "foo")).close();

    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
    makeSureCleanedUp(vols, service);
  }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
  public void testMRAsyncDiskService() throws Throwable {
 
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
   
    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
   
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
   
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();
   
    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
   
    // Move and delete them
    service.moveAndDeleteRelativePath(vols[0], a);
    assertFalse(fa.exists());
    service.moveAndDeleteRelativePath(vols[1], b);
    assertFalse(fb.exists());
    assertFalse(fc.exists());
   
    // asyncDiskService is NOT able to delete files outside all volumes.
    IOException ee = null;
    try {
      service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2");
    } catch (IOException e) {
      ee = e;
    }
    assertNotNull("asyncDiskService should not be able to delete files "
        + "outside all volumes", ee);
    // asyncDiskService is able to automatically find the file in one
    // of the volumes.
    assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d));
   
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
   */
  public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable {
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);

    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
   
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
   
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();

    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
   
    // Delete all of them
    service.cleanupAllVolumes();
   
    assertFalse(fa.exists());
    assertFalse(fb.exists());
    assertFalse(fc.exists());
    assertFalse(fd.exists());
View Full Code Here
284
285
286
287
288
289
290
291
292
293
294
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
   
    // Create the asyncDiskService which will delete all contents inside SUBDIR
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
   
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }
View Full Code Here
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
    this.conf.set("mapred.job.tracker.http.address",
        infoBindAddress + ":" + this.infoPort);
    LOG.info("JobTracker webserver: " + this.infoServer.getPort());

    // start async disk service for asynchronous deletion service
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(jobConf),
        jobConf.getLocalDirs(), conf);

    while (!Thread.currentThread().isInterrupted()) {
      try {
        // if we haven't contacted the namenode go ahead and do it
View Full Code Here
702
703
704
705
706
707
708
709
710
711
712
             ", actual max reduce slots = " + actualMaxReduceSlots);

    // Check local disk, start async disk service, and clean up all
    // local directories.
    checkLocalDirs(this.fConf.getLocalDirs());
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(fConf),
        fConf.getLocalDirs(), fConf);
    asyncDiskService.cleanupAllVolumes();
    DistributedCache.purgeCache(fConf, asyncDiskService);
    versionBeanName = VersionInfo.registerJMX("TaskTracker");
View Full Code Here
700
701
702
703
704
705
706
707
708
709
710
             ", actual max reduce slots = " + actualMaxReduceSlots);

    // Check local disk, start async disk service, and clean up all
    // local directories.
    checkLocalDirs(this.fConf.getLocalDirs());
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(fConf),
        fConf.getLocalDirs(), fConf);
    asyncDiskService.cleanupAllVolumes();
    DistributedCache.purgeCache(fConf, asyncDiskService);
    versionBeanName = VersionInfo.registerJMX("TaskTracker");
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
  public void testMRAsyncDiskService() throws Throwable {
 
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
   
    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
   
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
   
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();
   
    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
   
    // Move and delete them
    service.moveAndDeleteRelativePath(vols[0], a);
    assertFalse(fa.exists());
    service.moveAndDeleteRelativePath(vols[1], b);
    assertFalse(fb.exists());
    assertFalse(fc.exists());
   
    assertFalse(service.moveAndDeleteRelativePath(vols[1], "not_exists"));
   
    // asyncDiskService is NOT able to delete files outside all volumes.
    IOException ee = null;
    try {
      service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2");
    } catch (IOException e) {
      ee = e;
    }
    assertNotNull("asyncDiskService should not be able to delete files "
        + "outside all volumes", ee);
    // asyncDiskService is able to automatically find the file in one
    // of the volumes.
    assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d));
   
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.util.MRAsyncDiskService$DeleteTask

  • com.amazonaws.services.dynamodb.model.AttributeValue
  • com.amazonaws.services.dynamodb.model.DeleteItemRequest
  • com.amazonaws.services.dynamodb.model.Key
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.mapred.JobTracker
  • org.apache.hadoop.mapred.TaskTracker
  • org.apache.hadoop.mapreduce.util.TestMRAsyncDiskService
  • org.eclipse.core.resources.IContainer
  • org.eclipse.core.resources.IFile
  • org.eclipse.core.resources.IResource
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.