Package org.apache.hadoop.hdfs.server.common.Storage

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
     * directories
     */
    void ensureCurrentDirExists() throws IOException {
      for (Iterator<StorageDirectory> it
             = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File curDir = sd.getCurrentDir();
        if (!curDir.exists() && !curDir.mkdirs()) {
          throw new IOException("Could not create directory " + curDir);
        }
      }
    }
View Full Code Here
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
              + "_(\\d+)-(\\d+)_(\\d+)");
        }
      };
      Iterator<StorageDirectory> it = storage.dirIterator(NameNodeDirType.EDITS);
      for (;it.hasNext();) {
        StorageDirectory dir = it.next();
        File[] tempEdits = dir.getCurrentDir().listFiles(filter);
        if (tempEdits != null) {
          for (File t : tempEdits) {
            boolean success = t.delete();
            if (!success) {
              LOG.warn("Failed to delete temporary edits file: "
View Full Code Here
868
869
870
871
872
873
874
875
876
877
878
879
880
      @Override
      public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException {
        Iterator<StorageDirectory> iter = storage.dirIterator();
        while (iter.hasNext()) {
          StorageDirectory dir = iter.next();
          List<EditLogFile> editFiles = FileJournalManager.matchEditLogs(
              dir.getCurrentDir());
          for (EditLogFile f : editFiles) {
            if (f.getLastTxId() < minTxIdToKeep) {
              purger.purgeLog(f);
            }
          }
View Full Code Here
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
      @Override
      public void selectInputStreams(Collection<EditLogInputStream> streams,
          long fromTxId, boolean inProgressOk) {
        Iterator<StorageDirectory> iter = storage.dirIterator();
        while (iter.hasNext()) {
          StorageDirectory dir = iter.next();
          List<EditLogFile> editFiles;
          try {
            editFiles = FileJournalManager.matchEditLogs(
                dir.getCurrentDir());
          } catch (IOException ioe) {
            throw new RuntimeException(ioe);
          }
          FileJournalManager.addStreamsToCollectionFromFiles(editFiles, streams,
              fromTxId, inProgressOk);
View Full Code Here
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
      new HashMap<String, Map<File, StorageDirType>>();
   
    Map<File, StorageDirType> activeDirs = new HashMap<File, StorageDirType>();
    for (Iterator<StorageDirectory> it
        = getFSImage().getStorage().dirIterator(); it.hasNext();) {
      StorageDirectory st = it.next();
      activeDirs.put(st.getRoot(), st.getStorageDirType());
    }
    statusMap.put("active", activeDirs);
   
    List<Storage.StorageDirectory> removedStorageDirs
        = getFSImage().getStorage().getRemovedStorageDirs();
    Map<File, StorageDirType> failedDirs = new HashMap<File, StorageDirType>();
    for (StorageDirectory st : removedStorageDirs) {
      failedDirs.put(st.getRoot(), st.getStorageDirType());
    }
    statusMap.put("failed", failedDirs);
   
    return JSON.toString(statusMap);
  }
View Full Code Here
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
    ArrayList<File> newDirs = getDataDirsFromURIs(dataDirs);
    ArrayList<File> decomDirs = new ArrayList<File>();
 
    for (Iterator<StorageDirectory> storageIter = this.storage.dirIterator();
        storageIter.hasNext();) {
      StorageDirectory dir = storageIter.next();
     
      // Delete volumes not in service from DataStorage
      if (!((FSDataset)data).isValidVolume(dir.getCurrentDir())) {
        LOG.info("This dir is listed in conf, but not in service " + dir.getRoot());
        storageIter.remove();
        continue;
      }
 
      if (newDirs.contains(dir.getRoot())){
        // remove the dir already in-service in newDirs list
        LOG.info("This conf dir has already been in service " + dir.getRoot());
        newDirs.remove(dir.getRoot());
      } else {
        // add the dirs not described in conf files to decomDirs
        LOG.warn("The configuration does not contain serving dir " +
          dir.getRoot() + ", but we cannot remove it from serving volumes in current version." );
        decomDirs.add(dir.getRoot());
      }
    }
 
    if (newDirs.isEmpty()){
      LOG.info("All the configured dir is in service, and do not need refreshment.");
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
      // FS Image storage configuration
      out.print("<h3> " + nn.getRole() + " Storage: </h3>");
      out.print("<div class=\"dfstable\"> <table class=\"storage\" title=\"NameNode Storage\">\n"
              + "<thead><tr><td><b>Storage Directory</b></td><td><b>Type</b></td><td><b>State</b></td></tr></thead>");

      StorageDirectory st = null;
      for (Iterator<StorageDirectory> it
             = fsImage.getStorage().dirIterator(); it.hasNext();) {
        st = it.next();
        String dir = "" + st.getRoot();
        String type = "" + st.getStorageDirType();
        out.print("<tr><td>" + dir + "</td><td>" + type
            + "</td><td>Active</td></tr>");
      }

      long storageDirsSize = removedStorageDirs.size();
      for (int i = 0; i < storageDirsSize; i++) {
        st = removedStorageDirs.get(i);
        String dir = "" + st.getRoot();
        String type = "" + st.getStorageDirType();
        out.print("<tr><td>" + dir + "</td><td>" + type
            + "</td><td><span class=\"failed\">Failed</span></td></tr>");
      }

      out.print("</table></div>\n");
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
  public void testInprogressRecovery() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest0");
    // abort after the 5th roll
    NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()),
                                   5, new AbortSpec(5, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL,
                 getNumberOfTransactions(jm, 1, true, false));
  }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    // abort after the 5th roll
    NNStorage storage = setupEdits(editUris,
                                   5, new AbortSpec(5, 1));
    Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
    StorageDirectory sd = dirs.next();
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(6*TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
   
    sd = dirs.next();
    jm = new FileJournalManager(conf, sd, storage);
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
    NNStorage storage = setupEdits(editUris, 5,
                                   new AbortSpec(5, 0),
                                   new AbortSpec(5, 1),
                                   new AbortSpec(5, 2));
    Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
    StorageDirectory sd = dirs.next();
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(5*TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1,
        true, false));
   
    sd = dirs.next();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

  • org.apache.hadoop.hdfs.notifier.NotifierUtils
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderAvatar
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderTransactional
  • org.apache.hadoop.hdfs.notifier.server.TestServerLogReaderVersion
  • org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader
  • org.apache.hadoop.hdfs.qjournal.server.TestJournal
  • org.apache.hadoop.hdfs.server.common.StorageAdapter
  • org.apache.hadoop.hdfs.server.datanode.DataNode
  • org.apache.hadoop.hdfs.server.datanode.NameSpaceSliceStorage
  • org.apache.hadoop.hdfs.server.namenode.BackupImage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.