Package org.apache.hadoop.hdfs.server.namenode.FileJournalManager

Examples of org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile


      backup.stop();
      backup = null;
     
      // When shutting down the BN, it shouldn't finalize logs that are
      // still open on the NN
      EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
      assertEquals(editsLog.getFirstTxId(),
          nn.getFSImage().getEditLog().getCurSegmentTxId());
      assertTrue("Should not have finalized " + editsLog,
          editsLog.isInProgress());
     
      // do some edits
      assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
     
      // start a new backup node
View Full Code Here


      backup.stop();
      backup = null;
     
      // When shutting down the BN, it shouldn't finalize logs that are
      // still open on the NN
      EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
      assertEquals(editsLog.getFirstTxId(),
          nn.getFSImage().getEditLog().getCurSegmentTxId());
      assertTrue("Should not have finalized " + editsLog,
          editsLog.isInProgress());
     
      // do some edits
      assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
     
      // start a new backup node
View Full Code Here

      long segmentTxId, long expectedEndTxId)
      throws IOException {
    int numFinalized = 0;
    for (int i = 0; i < cluster.getNumNodes(); i++) {
      File logDir = cluster.getCurrentDir(i, JID);
      EditLogFile elf = FileJournalManager.getLogFile(logDir, segmentTxId);
      if (elf == null) {
        continue;
      }
      if (!elf.isInProgress()) {
        numFinalized++;
        if (elf.getLastTxId() != expectedEndTxId) {
          fail("File " + elf + " finalized to wrong txid, expected " +
              expectedEndTxId);
        }
      }     
    }
View Full Code Here

      long segmentTxId, long expectedEndTxId)
      throws IOException {
    int numFinalized = 0;
    for (int i = 0; i < cluster.getNumNodes(); i++) {
      File logDir = cluster.getCurrentDir(i, JID);
      EditLogFile elf = FileJournalManager.getLogFile(logDir, segmentTxId);
      if (elf == null) {
        continue;
      }
      if (!elf.isInProgress()) {
        numFinalized++;
        if (elf.getLastTxId() != expectedEndTxId) {
          fail("File " + elf + " finalized to wrong txid, expected " +
              expectedEndTxId);
        }
      }     
    }
View Full Code Here

    FSImage image = cluster.getNameNode().getFSImage();
    // it was set up to only have ONE StorageDirectory
    Iterator<StorageDirectory> it
      = image.storage.dirIterator(NameNodeDirType.EDITS);
    StorageDirectory sd = it.next();
    EditLogFile elf = FSImageTestUtil.findLatestEditsLog(sd);
    File ret = null;
    if(elf.getLastTxId() == HdfsConstants.INVALID_TXID)
      ret = NNStorage.getInProgressEditsFile(sd, elf.getFirstTxId());
    else
      ret = NNStorage.getFinalizedEditsFile(sd, elf.getFirstTxId(), elf.getLastTxId());
    assert ret.exists() : "expected " + ret + " exists";
    return ret.getAbsolutePath();
  }
View Full Code Here

        }

        // synchronized on journal so no files can be changed
        synchronized (task.journal) {
          // move away our local copy of the segment
          EditLogFile localCorruptedFile = null;
          try {
            localCorruptedFile = task.getInprogressSegment(remoteStartTxid);
            if (localCorruptedFile != null) {
              localCorruptedFile.moveAsideCorruptFile();
            }
          } catch (Exception e) {
            LOG.warn(logMsg + "exception when marking segment: "
                + localCorruptedFile + " as corrupt.", e);
          }
View Full Code Here

    List<String> logFilesDesc = mapper.readValue(json, type);

    // we need to convert the list of strings into edit log files
    List<EditLogFile> logFiles = new ArrayList<EditLogFile>();
    for (String lf : logFilesDesc) {
      logFiles.add(new EditLogFile(lf));
    }
    return logFiles;
  }
View Full Code Here

              + segmentTxId + ", first file found starts at txid: "
              + logFiles.get(0).getFirstTxId());
        }
       
        // the requested file is in the beginning of the list
        EditLogFile elf = logFiles.get(0);
        editFile = elf.getFile();
       
        long lastValidTxId;
        long maxValidLength;
       
        if (segmentTxId == journal.getCurrentSegmentTxId()) {
          if (!elf.isInProgress()) {
            throw new IOException(
                "Inconsistent state. Current segment must always be in progress.");
          }
          // this is the currently written segment, this will be the
          // most common scenario, where we want to tail this segment
          // we do not want to validate it at each request
          maxValidLength = journal.getValidSizeOfCurrentSegment();
          // we can safely read only up to this transaction
          // max - secures a degenerate case when we have multiple empty segments
          lastValidTxId = Math.max(segmentTxId - 1, journal.getCommittedTxnId());
        } else if (elf.isInProgress()) {
          // this is an in-progress segment, but there are newer segments already
          // (possibly in progress). This happens when the journal failed and was
          // restored at later time, we do not know what the last valid txid is.
          // we can still serve the file, just in case there is logic on the
          // reader side to handle this
          lastValidTxId = HdfsConstants.INVALID_TXID;
          maxValidLength = editFile.length();
        } else {
          // finalized segment has the last transaction id,
          // ad this transaction has been committed in the quorum
          // entire segment can be safely served
          // (segment cannot be finalized if the last transaction
          // was not synced to the quorum)
          lastValidTxId = elf.getLastTxId();
          maxValidLength = editFile.length();
        }
       
        if (position > maxValidLength) {
          throw new IOException("Position: " + position
              + " is beyond valid length of the file. File size: " + maxValidLength);
        }
        lengthToSend = maxValidLength - position;
       
        // we are sending lengthToSend bytes
        response.setHeader(TransferFsImage.CONTENT_LENGTH,
            String.valueOf(lengthToSend));
       
        // indicate that this stream can only serve up to this transaction
        response.setHeader(LAST_VALID_TXID, String.valueOf(lastValidTxId));
        response.setHeader(IS_IN_PROGRESS, String.valueOf(elf.isInProgress()));
              
        GetImageServlet.setFileNameHeaders(response, editFile);
       
        // prepare input stream
        RandomAccessFile rp = new RandomAccessFile(editFile, "r");   
View Full Code Here

      long segmentTxId, long expectedEndTxId)
      throws IOException {
    int numFinalized = 0;
    for (int i = 0; i < cluster.getNumNodes(); i++) {
      File logDir = cluster.getJournalCurrentDir(i, JID);
      EditLogFile elf = FileJournalManager.getLogFile(logDir, segmentTxId);
      if (elf == null) {
        continue;
      }
      if (!elf.isInProgress()) {
        numFinalized++;
        if (elf.getLastTxId() != expectedEndTxId) {
          fail("File " + elf + " finalized to wrong txid, expected " +
              expectedEndTxId);
        }
      }     
    }
View Full Code Here

      assertEquals(segments.size(), manifest.size());

      // for each segment we should have, find the corresponding entry in the
      // manifest
      for (SegmentDescriptor sd : segments) {
        EditLogFile elf = findFile(sd.startTxId, manifest);
        assertNotNull(elf);
        if (sd.endTxId == -1) {
          // manifest should have an entry for inprogress segment
          assertTrue(elf.isInProgress());
        } else {
          // manifest should conatin entry for a finalized segmet
          assertEquals(sd.endTxId, elf.getLastTxId());
          assertFalse(elf.isInProgress());
        }
      }
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.