Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.NNStorage


      GenericTestUtils.assertGlobEquals(currentDir, "edits_.*",
          NNStorage.getFinalizedEditsFileName(0, 1));
      // Make a truncated edits_2_inprogress
      File log = new File(currentDir,
          NNStorage.getInProgressEditsFileName(2));
      NNStorage storage = new NNStorage(conf,
                                        Collections.<URI>emptyList(),
                                        Lists.newArrayList(uri), null);
      if (updateTransactionIdFile) {
        storage.writeTransactionIdFileToStorage(2, null);
      }
      storage.close();

      new EditLogFileOutputStream(log, null).create();
      if (!inBothDirs) {
        break;
      }
View Full Code Here


   * Tests the getEditLogManifest function using mock storage for a number
   * of different situations.
   */
  @Test
  public void testEditLogManifestMocks() throws IOException {
    NNStorage storage;
    FSEditLog log;
    // Simple case - different directories have the same
    // set of logs, with an in-progress one at end
    storage = mockStorageWithEdits(
        "[1,100]|[101,200]|[201,]",
View Full Code Here

   */
  private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException {
    List<StorageDirectory> sds = Lists.newArrayList();
    List<URI> uris = Lists.newArrayList();

    NNStorage storage = Mockito.mock(NNStorage.class);
    for (String dirSpec : editsDirSpecs) {
      List<String> files = Lists.newArrayList();
      String[] logSpecs = dirSpec.split("\\|");
      for (String logSpec : logSpecs) {
        Matcher m = Pattern.compile("\\[(\\d+),(\\d+)?\\]").matcher(logSpec);
View Full Code Here

 
  public static NNStorage setupEdits(List<URI> editUris, int numrolls, boolean closeOnFinish,
                                     AbortSpec... abortAtRolls)
      throws IOException {
    List<AbortSpec> aborts = new ArrayList<AbortSpec>(Arrays.asList(abortAtRolls));
    NNStorage storage = new NNStorage(new Configuration(),
                                      Collections.<URI>emptyList(),
                                      editUris, null);
    storage.format();
    FSEditLog editlog = createEditLog(storage);;   
    // open the edit log and add two transactions
    // logGenerationStamp is used, simply because it doesn't
    // require complex arguments.
    editlog.open();
View Full Code Here

    File f1 = new File(TEST_DIR + "/alternatingjournaltest0");
    File f2 = new File(TEST_DIR + "/alternatingjournaltest1");

    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI());
   
    NNStorage storage = setupEdits(editUris, 10,
                                   new AbortSpec(1, 0),
                                   new AbortSpec(2, 1),
                                   new AbortSpec(3, 0),
                                   new AbortSpec(4, 1),
                                   new AbortSpec(5, 0),
                                   new AbortSpec(6, 1),
                                   new AbortSpec(7, 0),
                                   new AbortSpec(8, 1),
                                   new AbortSpec(9, 0),
                                   new AbortSpec(10, 1));
    long totaltxnread = 0;
    FSEditLog editlog = createEditLog(storage);;
    long startTxId = 0;
   
    // we need to finalize the inprogress segments (this is done on startup)
    editlog.recoverUnclosedStreams();
   
    List<EditLogInputStream> editStreams = new ArrayList<EditLogInputStream>();
    editlog.selectInputStreams(editStreams, startTxId, TXNS_PER_ROLL*11 - 1, 0);

    for (EditLogInputStream edits : editStreams) {
      FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(edits);
      long read = val.getNumTransactions();
      LOG.info("Loading edits " + edits + " read " + read);
      assertEquals(startTxId, val.getStartTxId());
      startTxId += read;
      totaltxnread += read;
    }

    editlog.close();
    storage.close();
    assertEquals(TXNS_PER_ROLL*11, totaltxnread);   
  }
View Full Code Here

  @Test
  public void testLoadingWithGaps() throws IOException {
    File f1 = new File(TEST_DIR + "/gaptest0");
    List<URI> editUris = ImmutableList.of(f1.toURI());

    NNStorage storage = setupEdits(editUris, 3);
   
    final long startGapTxId = 1*TXNS_PER_ROLL;
    final long endGapTxId = 2*TXNS_PER_ROLL - 1;

    File[] files = new File(f1, "current").listFiles(new FilenameFilter() {
View Full Code Here

  @Test
  public void testAutomaticRoll() throws IOException, InterruptedException {
    File f1 = new File(TEST_DIR + "/testAutomaticRoll");
    List<URI> editUris = ImmutableList.of(f1.toURI());

    NNStorage storage = setupEdits(editUris, 3);

    storage.format();
    Configuration conf = new Configuration();
    conf.setLong(FSEditLog.CONF_ROLL_TIMEOUT_MSEC, 200);
    FakeNameNode fakeNN = new FakeNameNode();
    AutomaticEditsRoller aer = new AutomaticEditsRoller(fakeNN);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.NNStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.