Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogOp


   
    for (int expected = firstTxnId;
        expected <= lastTxnId;
        expected++) {
     
      FSEditLogOp op = stream.readOp();
      while (op == null) {
        assertTrue("Expected to find txid " + expected + ", " +
            "but no more streams available to read from",
            iter.hasNext());
        stream = iter.next();
        op = stream.readOp();
      }
     
      assertEquals(FSEditLogOpCodes.OP_MKDIR, op.opCode);
      assertEquals(expected, op.getTransactionId());
    }
   
    assertNull(stream.readOp());
    assertFalse("Expected no more txns after " + lastTxnId +
        " but more streams are available", iter.hasNext());
View Full Code Here


   *         at the moment.
   * @throws IOException raised when a fatal error occurred.
   */
  public NamespaceNotification getNamespaceNotification()
      throws IOException {
    FSEditLogOp op = null;
    NamespaceNotification notification = null;
   
    // Keep looping until we reach an operation that can be
    // considered a notification.
    while (true) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("edits.size=" + editsFile.length() + " editsNew.size=" +
            editsNewFile.length());
      }
      try {
        op = inputStream.readOp();
        if (LOG.isDebugEnabled()) {
          LOG.debug("inputStream.readOP() returned " + op);
        }
      } catch (IOException e) {
        LOG.warn("inputStream.readOp() failed", e);
        tryReloadingEditLog();
        return null;
      } catch (Exception e2) {
        LOG.error("Error reading log operation", e2);
        throw new IOException(e2);
      }
     
      // No operation to read at the moment from the transaction log
      if (op == null) {
        core.getMetrics().reachedEditLogEnd.inc();
        handleNullRead();
        trySwitchingEditLog();
        return null;
      } else {
        core.getMetrics().readOperations.inc();
        readNullAfterStreamFinished = false;
      }
     
      if (LOG.isDebugEnabled()) {
        LOG.debug("Read operation: " + op + " with txId=" +
            op.getTransactionId());
      }
     
      if (ServerLogReaderUtil.shouldSkipOp(expectedTransactionId, op)) {
        updateStreamPosition();
        continue;
View Full Code Here

   * @throws IOException
   *           raised when a fatal error occurred.
   */
  @Override
  public NamespaceNotification getNamespaceNotification() throws IOException {
    FSEditLogOp op = null;
    NamespaceNotification notification = null;

    // Keep looping until we reach an operation that can be
    // considered a notification.
    while (true) {
     
      try {
        // if the stream is null, we need to setup next stream
        // if we cannot than this is a fatal failure
        refreshInputStream();

        // get current position in the stream
        currentEditLogInputStreamPosition = currentEditLogInputStream
            .getPosition();
       
        // try reading a transaction
        op = currentEditLogInputStream.readOp();
        InjectionHandler.processEventIO(InjectionEvent.SERVERLOGREADER_READOP);
        if (LOG.isDebugEnabled()) {
          LOG.info("inputStream.readOP() returned " + op);
        }
      } catch (Exception e) {
        // possibly ChecksumException
        LOG.warn("inputStream.readOp() failed", e);
        // try reopening current log segment
        tryReloadingEditLog();
        continue;
      }

      // for each successful read, we update state
      if (op == null) {
        // we can sleep to wait for new transactions
        sleep(500);
        // nothing in the edit log now
        core.getMetrics().reachedEditLogEnd.inc();
        checkProgress();
        continue;
      }

      if (ServerLogReaderUtil.shouldSkipOp(mostRecentlyReadTransactionTxId, op)){
        updateState(op, false);
        continue;
      }
      // update internal state, and check for progress
      updateState(op, true);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Read operation: " + op + " with txId="
          + (op == null ? "null" : op.getTransactionId()));
      }

      // Test if it can be considered a notification
      notification = ServerLogReaderUtil.createNotification(op);
      if (notification != null) {
View Full Code Here

            // start new segment
            int firstTxId = currentTxId;
            EditLogOutputStream out = jm.startLogSegment(currentTxId);

            // starting transaction
            FSEditLogOp startOp = new LogSegmentOp(
                FSEditLogOpCodes.OP_START_LOG_SEGMENT);
            startOp.setTransactionId(firstTxId);
            FSEditLogTestUtil.writeToStreams(startOp, out);
            LOG.info("Written op: " + startOp);
            writtenTransactions.add(startOp);

            currentTxId++;

            // other transactions
            List<FSEditLogOp> transactions = FSEditLogTestUtil
                .getContiguousLogSegment(currentTxId, currentTxId
                    + editsPerFile);

            for (int i = 0; i < editsPerFile; i++) {
              FSEditLogOp op = transactions.get(i);
              op.setTransactionId(currentTxId);
              FSEditLogTestUtil.writeToStreams(op, out);
              writtenTransactions.add(op);
              currentTxId++;
              LOG.info("Written op: " + op);
              if (i % 100 == 0) {
                Thread.sleep(10);
                FSEditLogTestUtil.flushStreams(out);
              }
            }

            // write ending transactions if needed
            if (endLogSegment || (segment == numSegments - 1)) {
              int lastTxId = currentTxId;
              FSEditLogOp endOp = new LogSegmentOp(
                  FSEditLogOpCodes.OP_END_LOG_SEGMENT);
              endOp.setTransactionId(lastTxId);
              FSEditLogTestUtil.writeToStreams(endOp, out);
              LOG.info("Written op: " + endOp);
              writtenTransactions.add(endOp);
              currentTxId++;
            }
View Full Code Here

    }

    @Override
    protected void _processEvent(InjectionEventI event, Object... args) {
      if (event == InjectionEvent.SERVERLOGREADER_UPDATE) {
        FSEditLogOp op = (FSEditLogOp) args[0];
        LOG.info("read transaction: " + op.getTransactionId());
        readTransactions.add(op);
      }
    }
View Full Code Here

    }
  }

  public static void writeOp(EditLogOutputStream stm, long txid)
      throws IOException {
    FSEditLogOp op = LogOpGenerator.getMkDirInstance("test");
    op.setTransactionId(txid);
    stm.write(op);
  }
View Full Code Here

   
    for (int expected = firstTxnId;
        expected <= lastTxnId;
        expected++) {
     
      FSEditLogOp op = stream.readOp();
      while (op == null) {
        assertTrue("Expected to find txid " + expected + ", " +
            "but no more streams available to read from",
            iter.hasNext());
        stream = iter.next();
        op = stream.readOp();
      }
     
      assertEquals(FSEditLogOpCodes.OP_MKDIR, op.opCode);
      assertEquals(expected, op.getTransactionId());
    }
   
    assertNull(stream.readOp());
    assertFalse("Expected no more txns after " + lastTxnId +
        " but more streams are available", iter.hasNext());
View Full Code Here

    for (int i = 1; i <= numEdits; i++) {
      assertEquals("Position in file must be equal position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
View Full Code Here

    for (int i = 0; i <= numEdits; i++) {
      assertEquals("Position in file must be equal to position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
View Full Code Here

      }
    }
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);
    for (int i = 0; i <= numEdits; i++) {
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.FSEditLogOp

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.