Package org.apache.hadoop.yarn.server.nodemanager.DeletionService

Examples of org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask

1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
     
      // first we will try to delete sub directories which are present. This
      // should then trigger parent directory to be deleted.
      List<Path> subDirs = buildDirs(r, dirs.get(0), 2);
     
      FileDeletionTask dependentDeletionTask =
          del.createFileDeletionTask(null, dirs.get(0), new Path[] {});
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      for (Path subDir : subDirs) {
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(null, null, new Path[] { subDir });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }

      int msecToWait = 20 * 1000;
      while (msecToWait > 0 && (lfs.util().exists(dirs.get(0)))) {
        Thread.sleep(100);
        msecToWait -= 100;
      }
      assertFalse(lfs.util().exists(dirs.get(0)));
     

      // Now we will try to delete sub directories; one of the deletion task we
      // will mark as failure and then parent directory should not be deleted.
      subDirs = buildDirs(r, dirs.get(1), 2);
      subDirs.add(new Path(dirs.get(1), "absentFile"));
     
      dependentDeletionTask =
          del.createFileDeletionTask(null, dirs.get(1), new Path[] {});
      deletionTasks = new ArrayList<FileDeletionTask>();
      for (Path subDir : subDirs) {
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(null, null, new Path[] { subDir });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      // marking one of the tasks as a failure.
      deletionTasks.get(2).setSuccess(false);
      for (FileDeletionTask task : deletionTasks) {
View Full Code Here
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
      this.baseDirIncludes = baseDirIncludes;
    }
   
    @Override
    public boolean matches(Object o) {
      FileDeletionTask fd = (FileDeletionTask)o;
      if (fd.getUser() == null && user != null) {
        return false;
      } else if (fd.getUser() != null && user == null) {
        return false;
      } else if (fd.getUser() != null && user != null) {
        return fd.getUser().equals(user);
      }
      if (!comparePaths(fd.getSubDir(), subDirIncludes)) {
        return false;
      }
      if (baseDirIncludes == null && fd.getBaseDirs() != null) {
        return false;
      } else if (baseDirIncludes != null && fd.getBaseDirs() == null ) {
        return false;
      } else if (baseDirIncludes != null && fd.getBaseDirs() != null) {
        if (baseDirIncludes.length != fd.getBaseDirs().size()) {
          return false;
        }
        for (int i =0 ; i < baseDirIncludes.length; i++) {
          if (!comparePaths(fd.getBaseDirs().get(i), baseDirIncludes[i])) {
            return false;
          }
        }
      }
      return true;
View Full Code Here
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
     
      // first we will try to delete sub directories which are present. This
      // should then trigger parent directory to be deleted.
      List<Path> subDirs = buildDirs(r, dirs.get(0), 2);
     
      FileDeletionTask dependentDeletionTask =
          del.createFileDeletionTask(null, dirs.get(0), new Path[] {});
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      for (Path subDir : subDirs) {
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(null, null, new Path[] { subDir });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }

      int msecToWait = 20 * 1000;
      while (msecToWait > 0 && (lfs.util().exists(dirs.get(0)))) {
        Thread.sleep(100);
        msecToWait -= 100;
      }
      assertFalse(lfs.util().exists(dirs.get(0)));
     

      // Now we will try to delete sub directories; one of the deletion task we
      // will mark as failure and then parent directory should not be deleted.
      subDirs = buildDirs(r, dirs.get(1), 2);
      subDirs.add(new Path(dirs.get(1), "absentFile"));
     
      dependentDeletionTask =
          del.createFileDeletionTask(null, dirs.get(1), new Path[] {});
      deletionTasks = new ArrayList<FileDeletionTask>();
      for (Path subDir : subDirs) {
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(null, null, new Path[] { subDir });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      // marking one of the tasks as a failure.
      deletionTasks.get(2).setSuccess(false);
      for (FileDeletionTask task : deletionTasks) {
View Full Code Here
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
      this.baseDirIncludes = baseDirIncludes;
    }
   
    @Override
    public boolean matches(Object o) {
      FileDeletionTask fd = (FileDeletionTask)o;
      if (fd.getUser() == null && user != null) {
        return false;
      } else if (fd.getUser() != null && user == null) {
        return false;
      } else if (fd.getUser() != null && user != null) {
        return fd.getUser().equals(user);
      }
      if (!comparePaths(fd.getSubDir(), subDirIncludes)) {
        return false;
      }
      if (baseDirIncludes == null && fd.getBaseDirs() != null) {
        return false;
      } else if (baseDirIncludes != null && fd.getBaseDirs() == null ) {
        return false;
      } else if (baseDirIncludes != null && fd.getBaseDirs() != null) {
        if (baseDirIncludes.length != fd.getBaseDirs().size()) {
          return false;
        }
        for (int i =0 ; i < baseDirIncludes.length; i++) {
          if (!comparePaths(fd.getBaseDirs().get(i), baseDirIncludes[i])) {
            return false;
          }
        }
      }
      return true;
View Full Code Here
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
  }

  private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
      Path userDirPath) throws IOException {
    RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
    FileDeletionTask dependentDeletionTask =
        del.createFileDeletionTask(null, userDirPath, new Path[] {});
    if (userDirStatus != null) {
      List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
      while (userDirStatus.hasNext()) {
        FileStatus status = userDirStatus.next();
        String owner = status.getOwner();
        FileDeletionTask deletionTask =
            del.createFileDeletionTask(owner, null,
              new Path[] { status.getPath() });
        deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
        deletionTasks.add(deletionTask);
      }
      for (FileDeletionTask task : deletionTasks) {
        del.scheduleFileDeletionTask(task);
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask

  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService
  • org.apache.hadoop.yarn.server.nodemanager.TestDeletionService
  • org.apache.hadoop.yarn.server.nodemanager.TestNodeManagerReboot$FileDeletionInclude
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.