Package org.apache.hadoop.hdfs.server.common

Examples of org.apache.hadoop.hdfs.server.common.StorageInfo

718
719
720
721
722
723
724
725
726
727
728
  DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) {
    DatanodeRegistration bpRegistration = createUnknownBPRegistration();
    String blockPoolId = nsInfo.getBlockPoolID();
   
    bpRegistration.setStorageID(getStorageId());
    StorageInfo storageInfo = storage.getBPStorage(blockPoolId);
    if (storageInfo == null) {
      // it's null in the case of SimulatedDataSet
      bpRegistration.storageInfo.layoutVersion = HdfsConstants.LAYOUT_VERSION;
      bpRegistration.setStorageInfo(nsInfo);
    } else {
View Full Code Here
878
879
880
881
882
883
884
885
886
887
888
889
        throw new IOException("Startup option not set.");
      }
      final String bpid = nsInfo.getBlockPoolID();
      //read storage info, lock data dirs and transition fs state if necessary
      storage.recoverTransitionRead(this, bpid, nsInfo, dataDirs, startOpt);
      final StorageInfo bpStorage = storage.getBPStorage(bpid);
      LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID()
          + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion()
          + ";nsInfo=" + nsInfo);
    }

    synchronized(this)  {
View Full Code Here
701
702
703
704
705
706
707
708
709
710
711
  DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) {
    DatanodeRegistration bpRegistration = createUnknownBPRegistration();
    String blockPoolId = nsInfo.getBlockPoolID();
   
    bpRegistration.setStorageID(getStorageId());
    StorageInfo storageInfo = storage.getBPStorage(blockPoolId);
    if (storageInfo == null) {
      // it's null in the case of SimulatedDataSet
      bpRegistration.storageInfo.layoutVersion = HdfsConstants.LAYOUT_VERSION;
      bpRegistration.setStorageInfo(nsInfo);
    } else {
View Full Code Here
861
862
863
864
865
866
867
868
869
870
871
872
        throw new IOException("Startup option not set.");
      }
      final String bpid = nsInfo.getBlockPoolID();
      //read storage info, lock data dirs and transition fs state if necessary
      storage.recoverTransitionRead(this, bpid, nsInfo, dataDirs, startOpt);
      final StorageInfo bpStorage = storage.getBPStorage(bpid);
      LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID()
          + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion()
          + ";nsInfo=" + nsInfo);
    }

    synchronized(this)  {
View Full Code Here
63
64
65
66
67
68
69
70
  /**
   * Create DatanodeRegistration
   */
  public DatanodeRegistration(String nodeName) {
    super(nodeName);
    this.storageInfo = new StorageInfo();
    this.exportedKeys = new ExportedAccessKeys();
  }
View Full Code Here
76
77
78
79
80
81
82
83
  public void setIpcPort(int ipcPort) {
    this.ipcPort = ipcPort;
  }

  public void setStorageInfo(DataStorage storage) {
    this.storageInfo = new StorageInfo(storage);
    this.storageID = storage.getStorageID();
  }
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.ROLLBACK);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(Integer.MIN_VALUE,
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         UpgradeUtilities.getCurrentFsscTime(cluster)));
      startDataNodeShouldFail(StartupOption.ROLLBACK);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
     
      log("DataNode rollback with newer fsscTime in previous", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.ROLLBACK);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         Long.MAX_VALUE));
      startDataNodeShouldFail(StartupOption.ROLLBACK);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode rollback with no edits file", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"edits"));
      }
      startNameNodeShouldFail(StartupOption.ROLLBACK);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode rollback with no image file", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"fsimage"));
      }
      startNameNodeShouldFail(StartupOption.ROLLBACK);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode rollback with corrupt version file", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      for (File f : baseDirs) {
        UpgradeUtilities.corruptFile(new File(f,"VERSION"));
      }
      startNameNodeShouldFail(StartupOption.ROLLBACK);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode rollback with old layout version in previous", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                         new StorageInfo(1,
                                                         UpgradeUtilities.getCurrentNamespaceID(null),
                                                         UpgradeUtilities.getCurrentFsscTime(null)));
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    } // end numDir loop
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    long fsscTimeOld = Long.MIN_VALUE;
    long fsscTimeCur = UpgradeUtilities.getCurrentFsscTime(null);
    long fsscTimeNew = Long.MAX_VALUE;
   
    return new StorageInfo[] {
      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeOld), // 0
      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeCur), // 1
      new StorageInfo(layoutVersionOld, namespaceIdCur, fsscTimeNew), // 2
      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeOld), // 3
      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeCur), // 4
      new StorageInfo(layoutVersionOld, namespaceIdOld, fsscTimeNew), // 5
      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeOld), // 6
      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeCur), // 7
      new StorageInfo(layoutVersionCur, namespaceIdCur, fsscTimeNew), // 8
      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeOld), // 9
      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeCur), // 10
      new StorageInfo(layoutVersionCur, namespaceIdOld, fsscTimeNew), // 11
      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeOld), // 12
      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeCur), // 13
      new StorageInfo(layoutVersionNew, namespaceIdCur, fsscTimeNew), // 14
      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeOld), // 15
      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeCur), // 16
      new StorageInfo(layoutVersionNew, namespaceIdOld, fsscTimeNew), // 17
    };
  }
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
                                                      new HdfsConfiguration());
    StorageInfo[] versions = initializeVersions();
    UpgradeUtilities.createStorageDirs(
                                       NAME_NODE, conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY), "current");
    cluster = new MiniDFSCluster(conf, 0, StartupOption.REGULAR);
    StorageInfo nameNodeVersion = new StorageInfo(
                                                  UpgradeUtilities.getCurrentLayoutVersion(),
                                                  UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                  UpgradeUtilities.getCurrentFsscTime(cluster));
    log("NameNode version info", NAME_NODE, null, nameNodeVersion);
    for (int i = 0; i < versions.length; i++) {
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
      log("DataNode upgrade with future stored layout version in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(Integer.MIN_VALUE,
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         UpgradeUtilities.getCurrentFsscTime(cluster)));
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
     
      log("DataNode upgrade with newer fsscTime in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         Long.MAX_VALUE));
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with no edits file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"edits"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with no image file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"fsimage"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with corrupt version file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        UpgradeUtilities.corruptFile(new File(f,"VERSION"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with old layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                         new StorageInfo(Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
                                                         UpgradeUtilities.getCurrentNamespaceID(null),
                                                         UpgradeUtilities.getCurrentFsscTime(null)));
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with future layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                         new StorageInfo(Integer.MIN_VALUE,
                                                         UpgradeUtilities.getCurrentNamespaceID(null),
                                                         UpgradeUtilities.getCurrentFsscTime(null)));
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    } // end numDir loop
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.common.StorageInfo

  • org.apache.hadoop.hdfs.DFSTestUtil
  • org.apache.hadoop.hdfs.notifier.NotifierUtils
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderAvatar
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderTransactional
  • org.apache.hadoop.hdfs.notifier.server.TestServerLogReaderVersion
  • org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader
  • org.apache.hadoop.hdfs.protocolPB.PBHelper
  • org.apache.hadoop.hdfs.protocolPB.TestPBHelper
  • org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$GetStorageStateProto
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.