Examples of org.apache.hadoop.hdfs.server.namenode.NNStorage

Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.NNStorage

org.apache.hadoop.hdfs.server.namenode.NNStorage
NNStorage is responsible for management of the StorageDirectories used by the NameNode.

    
    long imageTxId = proxy.getMostRecentCheckpointTxId();
    long curTxId = proxy.getTransactionID();
    
    // Format the storage (writes VERSION file)
    NNStorage storage = new NNStorage(conf, dirsToFormat, editUrisToFormat);
    storage.format(nsInfo);


    // Load the newly formatted image, using all of the directories (including shared
    // edits)
    FSImage image = new FSImage(conf);
    assert image.getEditLog().isOpenForRead() :

View Full Code Here

        "=====================================================");


    long imageTxId = proxy.getMostRecentCheckpointTxId();
    long curTxId = proxy.getTransactionID();
    
    NNStorage storage = new NNStorage(conf, dirsToFormat, editUrisToFormat);
    
    // Check with the user before blowing away data.
    if (!Storage.confirmFormat(storage.dirIterable(null),
            force, interactive)) {
      storage.close();
      return ERR_CODE_ALREADY_FORMATTED;
    }
    
    // Format the storage (writes VERSION file)
    storage.format(nsInfo);


    // Load the newly formatted image, using all of the directories (including shared
    // edits)
    FSImage image = new FSImage(conf);
    try {

View Full Code Here

      long txid = dfs.rollEdits();
      Assert.assertTrue(txid > 0);


      int retries = 0;
      while (++retries < 5) {
        NNStorage storage = dfsCluster.getNamesystem(1).getFSImage()
            .getStorage();
        if (storage.getFsImageName(txid - 1) != null) {
          return;
        }
        Thread.sleep(1000);
      }
      Assert.fail("new checkpoint does not exist");

View Full Code Here

   *
   * @return the created version file
   */
  public static File[] createNameNodeVersionFile(Configuration conf,
      File[] parent, StorageInfo version, String bpid) throws IOException {
    Storage storage = new NNStorage(conf, 
                              Collections.<URI>emptyList(), 
                              Collections.<URI>emptyList());
    storage.setStorageInfo(version);
    File[] versionFiles = new File[parent.length];
    for (int i = 0; i < parent.length; i++) {
      versionFiles[i] = new File(parent[i], "VERSION");
      StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
      storage.writeProperties(versionFiles[i], sd);
    }
    return versionFiles;
  }

View Full Code Here

  
  
  private void assertExistsInStorageDirs(MiniDFSCluster cluster,
      NameNodeDirType dirType,
      String filename) {
    NNStorage storage = cluster.getNamesystem().getFSImage().getStorage();
    for (StorageDirectory sd : storage.dirIterable(dirType)) {
      File f = new File(sd.getCurrentDir(), filename);
      assertTrue("Expect that " + f + " exists", f.exists());
    }
  }

View Full Code Here

      new EditLogFileOutputStream(log, 1024).create();
      if (!inBothDirs) {
        break;
      }
      
      NNStorage storage = new NNStorage(conf, 
          Collections.<URI>emptyList(),
          Lists.newArrayList(uri));
      
      if (updateTransactionIdFile) {
        storage.writeTransactionIdFileToStorage(3);
      }
      storage.close();
    }
    
    try {
      cluster = new MiniDFSCluster.Builder(conf)
        .numDataNodes(NUM_DATA_NODES).format(false).build();

View Full Code Here

   * Tests the getEditLogManifest function using mock storage for a number
   * of different situations.
   */
  @Test
  public void testEditLogManifestMocks() throws IOException {
    NNStorage storage;
    FSEditLog log;
    // Simple case - different directories have the same
    // set of logs, with an in-progress one at end
    storage = mockStorageWithEdits(
        "[1,100]|[101,200]|[201,]",

View Full Code Here

   */
  private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException {
    List<StorageDirectory> sds = Lists.newArrayList();
    List<URI> uris = Lists.newArrayList();


    NNStorage storage = Mockito.mock(NNStorage.class);
    for (String dirSpec : editsDirSpecs) {
      List<String> files = Lists.newArrayList();
      String[] logSpecs = dirSpec.split("\\|");
      for (String logSpec : logSpecs) {
        Matcher m = Pattern.compile("\\[(\\d+),(\\d+)?\\]").matcher(logSpec);

View Full Code Here

   * @param abortAtRolls Specifications for when to fail, see AbortSpec
   */
  public static NNStorage setupEdits(List<URI> editUris, int numrolls,
      boolean closeOnFinish, AbortSpec... abortAtRolls) throws IOException {
    List<AbortSpec> aborts = new ArrayList<AbortSpec>(Arrays.asList(abortAtRolls));
    NNStorage storage = new NNStorage(new Configuration(),
                                      Collections.<URI>emptyList(),
                                      editUris);
    storage.format(new NamespaceInfo());
    FSEditLog editlog = getFSEditLog(storage);    
    // open the edit log and add two transactions
    // logGenerationStamp is used, simply because it doesn't 
    // require complex arguments.
    editlog.initJournalsForWrite();

View Full Code Here

    File f1 = new File(TEST_DIR + "/alternatingjournaltest0");
    File f2 = new File(TEST_DIR + "/alternatingjournaltest1");


    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI());
    
    NNStorage storage = setupEdits(editUris, 10,
                                   new AbortSpec(1, 0),
                                   new AbortSpec(2, 1),
                                   new AbortSpec(3, 0),
                                   new AbortSpec(4, 1),
                                   new AbortSpec(5, 0),
                                   new AbortSpec(6, 1),
                                   new AbortSpec(7, 0),
                                   new AbortSpec(8, 1),
                                   new AbortSpec(9, 0),
                                   new AbortSpec(10, 1));
    long totaltxnread = 0;
    FSEditLog editlog = getFSEditLog(storage);
    editlog.initJournalsForWrite();
    long startTxId = 1;
    Iterable<EditLogInputStream> editStreams = editlog.selectInputStreams(startTxId, 
                                                                          TXNS_PER_ROLL*11);


    for (EditLogInputStream edits : editStreams) {
      FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(edits);
      long read = val.getNumTransactions();
      LOG.info("Loading edits " + edits + " read " + read);
      assertEquals(startTxId, val.getStartTxId());
      startTxId += read;
      totaltxnread += read;
    }


    editlog.close();
    storage.close();
    assertEquals(TXNS_PER_ROLL*11, totaltxnread);    
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.NNStorage

org.apache.hadoop.conf.Configuration

org.apache.hadoop.hdfs.notifier.NotifierUtils

org.apache.hadoop.hdfs.notifier.server.ServerLogReaderAvatar

org.apache.hadoop.hdfs.notifier.server.ServerLogReaderTransactional

org.apache.hadoop.hdfs.notifier.server.TestServerLogReaderVersion

org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader

org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby

org.apache.hadoop.hdfs.server.namenode.TestEditLog

org.apache.hadoop.hdfs.server.protocol.NamespaceInfo

org.apache.hadoop.hdfs.TestRollingUpgrade

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.