Package org.apache.hadoop.hdfs.server.namenode.FileJournalManager

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile

504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
    }

    // Paranoid sanity check: we should never overwrite a finalized log file.
    // Additionally, if it's in-progress, it should have at most 1 transaction.
    // This can happen if the writer crashes exactly at the start of a segment.
    EditLogFile existing = fjm.getLogFile(txid);
    if (existing != null) {
      if (!existing.isInProgress()) {
        throw new IllegalStateException("Already have a finalized segment " +
            existing + " beginning at " + txid);
      }
     
      // If it's in-progress, it should only contain one transaction,
      // because the "startLogSegment" transaction is written alone at the
      // start of each segment.
      existing.validateLog();
      if (existing.getLastTxId() != existing.getFirstTxId()) {
        throw new IllegalStateException("The log file " +
            existing + " seems to contain valid transactions");
      }
    }
   
View Full Code Here
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
   * @return the current state of the given segment, or null if the
   * segment does not exist.
   */
  private SegmentStateProto getSegmentInfo(long segmentTxId)
      throws IOException {
    EditLogFile elf = fjm.getLogFile(segmentTxId);
    if (elf == null) {
      return null;
    }
    if (elf.isInProgress()) {
      elf.validateLog();
    }
    if (elf.getLastTxId() == HdfsConstants.INVALID_TXID) {
      LOG.info("Edit log file " + elf + " appears to be empty. " +
          "Moving it aside...");
      elf.moveAsideEmptyFile();
      return null;
    }
    SegmentStateProto ret = SegmentStateProto.newBuilder()
        .setStartTxId(segmentTxId)
        .setEndTxId(elf.getLastTxId())
        .setIsInProgress(elf.isInProgress())
        .build();
    LOG.info("getSegmentInfo(" + segmentTxId + "): " + elf + " -> " +
        TextFormat.shortDebugString(ret));
    return ret;
  }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
        false,
        "/foo4/current/" + getImageFileName(122)));

    LogGroup lg = inspector.logGroups.get(123L);
    assertEquals(3, lg.logs.size());
    EditLogFile inProgressLog = lg.logs.get(2);
    assertTrue(inProgressLog.isInProgress());
   
    LoadPlan plan = inspector.createLoadPlan();

    // Check that it was marked corrupt.
    assertFalse(lg.logs.get(0).isCorrupt());
View Full Code Here
278
279
280
281
282
283
284
285
286
287
288
    assertFalse(lg.logs.get(0).isCorrupt());
    assertFalse(lg.logs.get(1).isCorrupt());
    assertTrue(lg.logs.get(2).isCorrupt());
   
    // Calling recover should move it aside
    EditLogFile badLog = lg.logs.get(2);
    Mockito.doNothing().when(badLog).moveAsideCorruptFile();
    Mockito.doNothing().when(lg.logs.get(0)).finalizeLog();
    Mockito.doNothing().when(lg.logs.get(1)).finalizeLog();
   
    lg.recover();
View Full Code Here
301
302
303
304
305
306
307
308
309
310
311
312
313
314
      String path, int numValidTransactions) throws IOException {
   
    for (LogGroup lg : inspector.logGroups.values()) {
      List<EditLogFile> logs = lg.logs;
      for (int i = 0; i < logs.size(); i++) {
        EditLogFile log = logs.get(i);
        if (log.getFile().getPath().equals(path)) {
          // mock out its validation
          EditLogFile spyLog = spy(log);
          doReturn(new FSEditLogLoader.EditLogValidation(-1, numValidTransactions))
            .when(spyLog).validateLog();
          logs.set(i, spyLog);
          return;
        }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
      backup.stop();
      backup = null;
     
      // When shutting down the BN, it shouldn't finalize logs that are
      // still open on the NN
      EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
      assertEquals(editsLog.getFirstTxId(),
          nn.getFSImage().getEditLog().getCurSegmentTxId());
      assertTrue("Should not have finalized " + editsLog,
          editsLog.isInProgress());
     
      // do some edits
      assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
     
      // start a new backup node
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
      String[] args = new String[]{"-saveNamespace"};

      // verify that the edits file is NOT empty
      NameNode nn = cluster.getNameNode();
      for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) {
        EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd);
        assertTrue(log.isInProgress());
        assertEquals("In-progress log " + log + " should have 5 transactions",
            5, log.validateLog().numTransactions);
      }

      // Saving image in safe mode should succeed
      fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
      try {
        admin.run(args);
      } catch(Exception e) {
        throw new IOException(e.getMessage());
      }
      // verify that the edits file is empty except for the START txn
      for (StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) {
        EditLogFile log = FSImageTestUtil.findLatestEditsLog(sd);
        assertTrue(log.isInProgress());
        assertEquals("In-progress log " + log + " should only have START txn",
            1, log.validateLog().numTransactions);
      }

      // restart cluster
      cluster.shutdown();
      cluster = null;
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
      backup.stop();
      backup = null;
     
      // When shutting down the BN, it shouldn't finalize logs that are
      // still open on the NN
      EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
      assertEquals(editsLog.getFirstTxId(),
          nn.getFSImage().getEditLog().getCurSegmentTxId());
      assertTrue("Should not have finalized " + editsLog,
          editsLog.isInProgress());
     
      // do some edits
      assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
     
      // start a new backup node
View Full Code Here
138
139
140
141
142
143
144
145
146
147
   
    this.fjm = storage.getJournalManager();
   
    this.metrics = JournalMetrics.create(this);
   
    EditLogFile latest = scanStorageForLatestEdits();
    if (latest != null) {
      highestWrittenTxId = latest.getLastTxId();
    }
  }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
   
    LOG.info("Scanning storage " + fjm);
    List<EditLogFile> files = fjm.getLogFiles(0);
   
    while (!files.isEmpty()) {
      EditLogFile latestLog = files.remove(files.size() - 1);
      latestLog.validateLog();
      LOG.info("Latest log is " + latestLog);
      if (latestLog.getLastTxId() == HdfsConstants.INVALID_TXID) {
        // the log contains no transactions
        LOG.warn("Latest log " + latestLog + " has no transactions. " +
            "moving it aside and looking for previous log");
        latestLog.moveAsideEmptyFile();
      } else {
        return latestLog;
      }
    }
   
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile

  • org.apache.hadoop.hdfs.qjournal.client.TestQJMRecovery
  • org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.server.GetJournalEditServlet
  • org.apache.hadoop.hdfs.qjournal.server.Journal
  • org.apache.hadoop.hdfs.qjournal.server.JournalNodeJournalSyncer
  • org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector$LogGroup
  • org.apache.hadoop.hdfs.server.namenode.OfflineEditsViewerHelper
  • org.apache.hadoop.hdfs.server.namenode.TestBackupNode
  • org.apache.hadoop.hdfs.server.namenode.TestCheckPointForSecurityTokens
  • org.apache.hadoop.hdfs.server.namenode.TestFSImageStorageInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.