* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This is an implementation of the AvatarNode, a hot

* standby for the NameNode.

* This is really cool, believe me!

* The AvatarNode has two avatars.. the Standby avatar and the Active

* avatar.

*

* In the Standby avatar, the AvatarNode is consuming transaction logs

* generated by the primary (via a transaction log stored in a shared device).

* Typically, the primary Namenode is writing transactions to a NFS filesystem

* and the Standby is reading the log from the same NFS filesystem. The

* Standby is also making periodic checkpoints to the primary namenode.

*

* A manual command can switch the AvatarNode from the Standby avatar

* to the Active avatar. In the Active avatar, the AvatarNode performs precisely

* the same functionality as a real usual Namenode. The switching from

* Standby avatar to the Active avatar is fast and can typically occur

* within seconds.

*

* Typically, an adminstrator will run require two shared mount points for

* transaction logs. It has to be set in fs.name.dir.shared0 and

* fs.name.dir.shared1 (similarly for edits). Then the adminstrator starts

* the AvatarNode on two different machines as follows:

*

* bin/hadoop org.apache.hadoop.hdfs.server.namenode.AvatarNode -zero -active

* bin/hadoop org.apache.hadoop.hdfs.server.namenode.AvatarNode -one -standby

* The first  AvatarNode uses  fs.name.dir.shared0 while the second

* AvatarNode uses fs.name.dir.shared1 to write its transaction logs.

* Also, at startup, the first instance is the primary Namenode and the

* second instance is the Standby

*

* After a while, the adminstrator decides to change the avatar of the

* second instance to Active. In this case, he/she has to first ensure that the

* first instance is really really dead. This code does not handle the

* split-brain scenario where there are two active namenodes in one cluster.

*

*/

    implements AvatarProtocol, AvatarNodeStatusMBean {

  static {

    Configuration.addDefaultResource("avatar-default.xml");

    Configuration.addDefaultResource("avatar-site.xml");

  }

  public static final Log LOG = LogFactory.getLog(AvatarNode.class.getName());

  private static final int    INVALIDATES_CLEANUP_INTERVAL = 60 * 1000;

  private static final String STORAGE_FILE_LOCK     = "in_use.lock";

  private static final String EDITSFILE     = "/current/edits";

  private static final String EDITSNEW     = "/current/edits.new";

  private static final String TIMEFILE     = "/current/fstime";

  private static final String IMAGEFILE = "/current/fsimage";

  private static final String IMAGENEW     ="/current/fsimage.ckpt";

  public static final long TXID_IGNORE = -1;

  public static final String FAILOVER_SNAPSHOT_FILE = "failover_snapshot_file";

  static final SimpleDateFormat dateForm =

    new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss.SSS");

  // The instanceId is assigned at startuptime and does not change for

  // the lifetime of the Node. The adminstrator has to name each instance

  // of the AvatarNode with a different instanceId. The node number is used

  // by the AvaterNode to determine which shared devices it should use to

  // checkpoint the image.

  //

  private InstanceId instance;

  // The time when (and if) the fsimage was sync-ed from the remote AvatarNode

  volatile private long startCheckpointTime;

  private Server server;                   /** RPC server */

  private InetSocketAddress serverAddress; /** RPC server address */

  private volatile Avatar currentAvatar;            // the current incarnation of this node

  private Standby standby;                 // the standby object

  private Configuration confg;             // config for the standby namenode

  private Configuration startupConf;       // config for the namenode

  private Thread standbyThread;            // the standby daemon thread

  private InvalidatesCleaner cleaner;      // The thread cleaning up invalidates

  private Thread cleanerThread;

  private RunInfo runInfo;

  private long sessionId;

  private StandbySafeMode standbySafeMode;

  private volatile boolean isInitialized = false;

  protected final boolean enableTestFramework; 

  protected final boolean enableTestFrameworkFsck;

 

  private String failoverFsck = "";

  /**

   * The startup Conf is the original configuration of the AvatarNode. It is used by the

   * secondary namenode to talk to the primary namenode.

   * The conf is the modified configuration that is used by the standby namenode

   */

  AvatarNode(Configuration startupConf, Configuration conf,

             StartupInfo startInfo, RunInfo runInfo, long sessionId) throws IOException {

    super(conf);   

    this.sessionId = sessionId;

    this.runInfo = runInfo;

    this.instance = startInfo.instance;

    this.enableTestFramework =

      (conf.getFloat("dfs.avatarnode.failover.sample.percent", 0.0f) != 0.0f);

    this.enableTestFrameworkFsck =

        (conf.getBoolean("dfs.avatarnode.failover.fsck", false));

   

    // if we are starting as the standby then

    // record the fstime of the checkpoint that we are about to sync from

    if (startInfo.isStandby) {

      // Set the checkpoint time to the fstime of the image and edits

      // that were copied

      setStartCheckpointTime(readLocalFstime(conf));

    }

    initialize(conf);

    currentAvatar = startInfo.isStandby ? Avatar.STANDBY : Avatar.ACTIVE;

    this.startupConf = startupConf;

    this.confg = conf;

    this.nameserviceId = startInfo.serviceName;

    if (currentAvatar == Avatar.STANDBY) {

      // Verify we have the correct safemode.

      SafeModeInfo safeMode = super.namesystem.getSafeModeInstance();

      if (safeMode == null || !(safeMode instanceof StandbySafeMode)) {

        throw new IOException("Invalid safe mode for Standby Avatar : "

            + safeMode + " Standby Avatar should be using "

            + StandbySafeMode.class + " as its dfs.safemode.impl");

      }

      standbySafeMode = (StandbySafeMode) safeMode;

      // Standby has a different property for the max buffered transactions

      // to replay the log faster

      int maxStandbyBufferedTransactions =

        confg.getInt("dfs.max.standby.buffered.transactions",

            HdfsConstants.DEFAULT_MAX_BUFFERED_TRANSACTIONS);

      FSEditLog.setMaxBufferedTransactions(maxStandbyBufferedTransactions);

      // Create a standby object which does the actual work of

      // processing transactions from the primary and checkpointing

      standby = new Standby(this, startupConf, confg);

      standbyThread = new Thread(standby);

      standbyThread.start();

      cleaner = new InvalidatesCleaner();

      cleanerThread = new Thread(cleaner);

      cleanerThread.start();

    }

    isInitialized = true;

  }

 

  protected void setFailoverFsck(String fsck) {

    failoverFsck = fsck;

  }

  /**

   * Generates a new session id for the cluster and writes it to zookeeper. Some

   * other data in zookeeper (like the last transaction id) is written to

   * zookeeper with the sessionId so that we can easily determine in which

   * session was this data written. The sessionId is unique since it uses the

   * current time.

   *

   * @return the session id that it wrote to ZooKeeper

   * @throws IOException

   */

  private static long writeSessionIdToZK(Configuration conf) throws IOException {

    long ssid = -1;

    int maxTries = conf.getInt("dfs.avatarnode.sync.ssidtxid.retries", 3);

    // Whether or not to verify the sessionid after writing it to ZK.

    int tries = 0;

    while (tries < maxTries) {

      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(conf, null);

      try {

        ssid = now();

        zk.registerPrimarySsId(getClusterAddress(conf), ssid);

        // Be extra careful and verify the data was synced to zk.

        Long ssIdInZk = zk.getPrimarySsId(getClusterAddress(conf));

        if (ssid != ssIdInZk) {

          throw new IOException("Session Id in the NameNode : " + ssid +

              " does not match the session Id in Zookeeper : " + ssIdInZk);

        }

        break;

      } catch(Exception e) {

        if (tries == maxTries - 1 ) {

          throw new IOException(e);

        }

      } finally {

        try {

          zk.shutdown();

        } catch (InterruptedException ie) {

          if (tries == maxTries - 1) {

            throw new IOException(ie);

          }

        }

      }

    }

    return ssid;

  }

  /**

   * Wait for the StandbyNode to exit. If it does, then stop the underlying namenode.

   */

  void waitForRestart() {

    if (standbyThread != null) {

      try {

        // if this is the standby avatarnode, then wait for the Standby to exit

        standbyThread.join();

      } catch (InterruptedException ie) {

        //eat it up

      }

      standbyThread = null;

      LOG.info("waitForRestart Standby thread exited.");

      // if we are still in standbymode, that means we need to restart from scratch.

      if (getAvatar() == Avatar.STANDBY) {

        runInfo.isRunning = false;

        LOG.info("waitForRestart Stopping encapsulated namenode.");

        super.stop();            // terminate encapsulated namenode

        super.join();            // wait for encapsulated namenode to exit

        shutdownStandby();

        LOG.info("waitForRestart exiting");

        return;

      }

    }

    super.join();            // wait for encapsulated namenode

  }

  public void registerMBean() {

    StandardMBean avatarNodeBean;

    try {

      avatarNodeBean = new StandardMBean(this, AvatarNodeStatusMBean.class);

      MBeanUtil.registerMBean("AvatarNode", "AvatarNodeState", avatarNodeBean);

    } catch (NotCompliantMBeanException mex) {

      LOG.error("Error registering mbean with JMX", mex);

    }

  }

 

  @Override

  public String getInstance() {

    return this.instance.toString();

  }

 

  @Override

  public String getState() {

    return this.currentAvatar.toString();

  }

 

  @Override

  public long getLagBytes() {

    if (this.standby == null)

    return 0;

    return this.standby.getLagBytes();

  }

  public Configuration getStartupConf() {

    return this.startupConf;

  }

  /**

   * Initialize AvatarNode

   * @param conf the configuration

   */

  private void initialize(Configuration conf) throws IOException {

    InetSocketAddress socAddr = AvatarNode.getAddress(conf);

    int handlerCount = conf.getInt("hdfs.avatarnode.handler.count", 3);

    // create rpc server

    this.server = RPC.getServer(this, socAddr.getHostName(),

                                socAddr.getPort(),

                                handlerCount, false, conf);

    // The rpc-server port can be ephemeral... ensure we have the

    // correct info

    this.serverAddress = this.server.getListenerAddress();

    LOG.info("AvatarNode up at: " + this.serverAddress);

    this.registerMBean();

    this.server.start();

  }

  /**

   * If the specified protocol is AvatarProtocol, then return the

   * AvatarProtocol version id, otherwise delegate to the underlying

   * namenode.

   */

  public long getProtocolVersion(String protocol,

                                 long clientVersion) throws IOException {

    if (protocol.equals(AvatarProtocol.class.getName())) {

      return AvatarProtocol.versionID;

    } else {

      return super.getProtocolVersion(protocol, clientVersion);

    }

  }

  //

  // methods to support Avatar Protocol

  //

  /**

   * @inheritDoc

   */

  public synchronized Avatar getAvatar() {

    return currentAvatar;

  }

 

  /**

   * @inheritDoc

   */

  public Avatar reportAvatar() {

    return currentAvatar;

  }

 

  /**

   * @inheritDoc

   */

  public boolean isInitialized() {

    return isInitialized;

  }

  private static class ShutdownAvatarThread extends Thread {

    private final AvatarNode node;

    public ShutdownAvatarThread(AvatarNode node) {

      this.node = node;

    }

    public void run() {

      try {

        node.runInfo.shutdown = true;

        LOG.info("Shutdown thread for: " + node.currentAvatar + " starting...");

        if (node.currentAvatar == Avatar.STANDBY) {

          // make sure that all transactions are consumed

          try {

            node.standby.quiesce(AvatarNode.TXID_IGNORE);

          } catch (Throwable e) {

            LOG.warn("Standby: ", e);

          }

        }

        // Need to stop RPC threads before capturing any final data about the

        // primary avatar.

        node.stopRPC(false);

        long totalBlocks = node.namesystem.getBlocksTotal();

        String fsck = "";

        try {

          if (node.enableTestFramework

              && node.enableTestFrameworkFsck) {

            LOG.info("Failover: Test framework - running fsck");

            fsck = node.runFailoverFsck();

            LOG.info("Failover: Test framework - fsck done");

          }

        } catch (IOException e) { /*ignore*/ }

       

        // after quiescing all communication and joining

        // all threads, we should still have all streams available

        node.verifyEditStreams();

       

        // stop the node (namesystem, fsimage, editlog, etc.)

        node.stop();

        node.join();            // wait for encapsulated namenode to exit

       

        if (InjectionHandler.falseCondition(InjectionEvent.AVATARNODE_SHUTDOWN)) {

          // simulate crash

          return;

        }

       

        if (node.currentAvatar == Avatar.STANDBY) {

          node.shutdownStandby();

        } else if (node.currentAvatar == Avatar.ACTIVE) {

          // If we are the primary we need to sync our last transaction id to

          // zookeeper.

          node.writeFailoverTestData(fsck);

          node.writeLastTxidToZookeeper(totalBlocks);

        }

      } catch (Exception e) {

        LOG.error("shutdownAvatar() failed", e);

      }

    }

  }

  private void verifyEditStreams() throws IOException {

    if (currentAvatar == Avatar.STANDBY)

      return;

    int expectedEditStreams = FSNamesystem.getNamespaceEditsDirs(confg).size();

    int actualEditStreams = this.namesystem.getFSImage().getEditLog()

        .getNumEditStreams();

    if (expectedEditStreams != actualEditStreams

        || InjectionHandler

            .falseCondition(InjectionEvent.AVATARNODE_CHECKEDITSTREAMS)) {

      String msg = "Failover: Cannot proceed - number of required edit streams: "

          + expectedEditStreams + " current number: " + actualEditStreams;

      LOG.fatal(msg);

      throw new IOException(msg);

    }

  }

 

  /**

   * Shuts down the avatar node

   * @param synchronous - should the function wait for the shutdown to complete

   * @throws IOException

   */

  public synchronized void shutdown(boolean synchronous) throws IOException {

    LOG.info("Shutdown: Asynchronous shutdown for: " + currentAvatar);

   

    if (runInfo.shutdown) {

      LOG.info("Shutdown: Node already shut down");

      return;

    }

    runInfo.shutdown = true;

   

    verifyEditStreams();

    Thread shutdownThread = new ShutdownAvatarThread(this);

    shutdownThread.setName("ShutDown thread for : " + serverAddress);

    shutdownThread.setDaemon(false);

    shutdownThread.start();

   

    if (synchronous) {

      LOG.info("Shutdown: Waiting for shutdown to complete");

      try {

        shutdownThread.join();

      } catch (InterruptedException ie) {

        throw new IOException(ie);

      }

    }

  }

 

  @Override

  public void shutdownAvatar() throws IOException {

    shutdown(false);

  }

 

  /**

   * Used only for testing.

   */

  public void doCheckpoint() throws IOException {

    if (currentAvatar != Avatar.STANDBY) {

      throw new IOException("This is not the standby avatar");

    }

    standby.doCheckpoint();

  }

 

  /**

   * Used only for testing.

   */

  public Standby getStandby() throws IOException {

    if (currentAvatar != Avatar.STANDBY) {

      throw new IOException("This is not the standby avatar");

    }

    return standby;

  }

  public long getSessionId() throws IOException {

    if (currentAvatar != Avatar.ACTIVE) {

      throw new IOException("This is not the primary avatar");

    }

    return this.sessionId;

  }

  /**

   * Used only for testing.

   */

  public void quiesceStandby(long txId) throws IOException {

    if (currentAvatar != Avatar.STANDBY) {

      throw new IOException("This is not the standby avatar");

    }

    standby.quiesce(txId);

  }

  public static String getClusterAddress(Configuration conf)

    throws UnknownHostException {

    InetSocketAddress addr = NameNode.getClientProtocolAddress(conf);

    return addr.getHostName() + ":" + addr.getPort();

  }

  /**

   * Writes the last transaction id of the primary avatarnode to zookeeper.

   *

   * @throws IOException

   */

  private void writeLastTxidToZookeeper(long totalBlocks) throws IOException {

    long lastTxid = super.getLastWrittenTxId();

    LOG.info("Failover - writing lastTxId: " + lastTxid + ", total blocks: " + totalBlocks);

    if (lastTxid < 0) {

      LOG.warn("Invalid last transaction id : " + lastTxid

          + " skipping write to zookeeper.");

      return;

    }

    ZookeeperTxId zkTxid = new ZookeeperTxId(this.sessionId, lastTxid,

        totalBlocks);

    int maxTries = startupConf.getInt("dfs.avatarnode.sync.ssidtxid.retries", 3);

    int tries = 0;

    while (true) {

      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);

      try {

        zk.registerLastTxId(getClusterAddress(this.startupConf), zkTxid);

        return;

      } catch (Exception e) {

        if (tries > maxTries) {

          throw new IOException(e);

        } else {

          tries++;

          LOG.warn("Error syncing last txid to zk, retrying ....", e);

          try {

            Thread.sleep(5000);

          } catch (InterruptedException ie) {

            throw new IOException("writeLastTxidToZookeeper() interrupted", ie);

          }

        }

      } finally {

        try {

          zk.shutdown();

        } catch (InterruptedException ie) {

          throw new IOException(ie);

        }

      }

    }

  }

  public void shutdownStandby() {

    standby.shutdown();

    if (server != null) {    // shutdown the AvatarNode

      LOG.info("Stopping avatarnode rpcserver.");

      server.stop();

      try {

        server.join();

      } catch (InterruptedException ie) {

        //eat it up

      }

    }

    if (cleaner != null) {

      // Shut down the cleaner thread as it will keep

      // the process from shutting down

      cleaner.stop();

      cleanerThread.interrupt();

      try {

        cleanerThread.join();

      } catch (InterruptedException iex) {

        Thread.currentThread().interrupt();

      }

    }

  }

 

  /**

   * Stops all RPC threads and ensures that all RPC handlers have exited.

   * Stops all communication to the namenode.

   */

  protected void stopRPC(boolean interruptClientHadlers) throws IOException {

    try {

      super.stopRPC(interruptClientHadlers);

      LOG.info("stopRPC: Stopping avatardatanode server");

      this.server.stop(interruptClientHadlers);

      this.server.waitForHandlers();

    } catch (InterruptedException ex) {

      throw new IOException("stopRPC() interrupted", ex);

    }

  }

  private ZookeeperTxId getLastTransactionId() throws IOException {

    try {

      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);

      try {

        // Gather session id and transaction id data.

        String address = getClusterAddress(this.startupConf);

        long sessionId = zk.getPrimarySsId(address);

        ZookeeperTxId zkTxId = zk.getPrimaryLastTxId(address);

        if (sessionId != zkTxId.getSessionId()) {

          throw new IOException("Session Id in the ssid node : " + sessionId

              + " does not match the session Id in the txid node : "

              + zkTxId.getSessionId());

        }

        return zkTxId;

      } finally {

        zk.shutdown();

      }

    } catch (Exception e) {

      throw new IOException(e);

    }

  }

  private void verifyTransactionIds(ZookeeperTxId zkTxId) throws IOException {

    long zkLastTxId = zkTxId.getTransactionId();

    long totalBlocks = zkTxId.getTotalBlocks();

    long lastTxId = super.getLastWrittenTxId();

    // Verify transacation ids.

    if (lastTxId < 0 || zkLastTxId < 0) {

      throw new IOException("Invalid transacation ids, txid in NameNode : "

          + lastTxId + " txid in Zookeeper : " + zkLastTxId);

    } else if (lastTxId != zkLastTxId) {

      throw new IOException("The transacation id in the namenode : "

          + lastTxId + " does not match the transaction id in zookeeper : "

          + zkLastTxId);

    } else if (totalBlocks != super.namesystem.getBlocksTotal()) {

      throw new IOException("Total blocks in ZK : " + totalBlocks

          + " don't match up with total blocks on Standby : "

          + super.namesystem.getBlocksTotal());

    }

  }

  private void registerAddressToZK(AvatarZooKeeperClient zk, String confParam)

      throws IOException {

    String address = startupConf.get(confParam);

    String realAddress = confg.get(confParam);

    if (address != null && realAddress != null) {

      zk.registerPrimary(address, realAddress, true);

    }

  }

  private void registerAsPrimaryToZK() throws IOException {

    // Register client port address.

    String address = getClusterAddress(startupConf);

    String realAddress = getClusterAddress(confg);

    AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);

    try {

      zk.registerPrimary(address, realAddress, true);

      // Register dn protocol address

      registerAddressToZK(zk, "dfs.namenode.dn-address");

      // Register http address

      registerAddressToZK(zk, "dfs.http.address");

      // Register rpc address

      registerAddressToZK(zk, AvatarNode.DFS_NAMENODE_RPC_ADDRESS_KEY);

    } finally {

      try {

        zk.shutdown();

      } catch (InterruptedException e) {

        throw new IOException("Could not shutdown zk client", e);

      }

    }

  }

  private File getSnapshotFile(Configuration conf) throws IOException {

    return new File(getRemoteEditsFile(conf).getParentFile(),

        FAILOVER_SNAPSHOT_FILE);

  }

  private void writeFailoverTestData(String fsck) throws IOException {

    if (!enableTestFramework) {

      LOG.info("Failover: Test framework - disabled");

      return;

    } 

    float samplePercent = confg.getFloat(

        "dfs.avatarnode.failover.sample.percent", 0.05f);

    LOG.info("Failover: Test framework - using " + (100.0*samplePercent)

        + " % sample size");     

    List<FileStatusExtended> stat = super.getRandomFilesSample(samplePercent);

    AvatarFailoverSnapshot snapshot = new AvatarFailoverSnapshot(

        super.namesystem.getOpenFiles(), stat);

    File snapshotFile = new File(

        getSharedEditsFile(confg).getParentFile(), FAILOVER_SNAPSHOT_FILE);

    DataOutputStream out = new DataOutputStream(

        new BufferedOutputStream(new FileOutputStream(snapshotFile))); 

    try {

      snapshot.write(out);

      out.writeBoolean(enableTestFrameworkFsck);

      if (enableTestFrameworkFsck) {

        Text.writeString(out, fsck);

      }

    } finally {

      out.close();

    } 

    LOG.info("Failover: Test framework - saved snapshot file : " + snapshotFile);

  }

  private void verifySnapshotSampledFile(FileStatusExtended file)

      throws IOException {

    FileStatusExtended stat = super.namesystem.getFileInfoExtended(file

        .getPath().toString());

    if (!stat.equals(file)) {

      throw new IOException("Information for file : " + file.getPath()

          + " does not match with information on snapshot file, expected : "

          + file + ", actual : " + stat);

    }

  }

  private void verifyOpenFiles(OpenFilesInfo openFilesInfo) throws IOException {

    if (openFilesInfo.getGenStamp() != super.namesystem.getGenerationStamp()) {

      throw new IOException(

          "GS on snapshot file doesn't match with GS on node : "

              + openFilesInfo.getGenStamp() + ", "

              + super.namesystem.getGenerationStamp());

    }

    for (FileStatusExtended stat : openFilesInfo.getOpenFiles()) {

      verifySnapshotSampledFile(stat);

    }

  }

  private String verifyFailoverTestData() throws IOException {

    if (!enableTestFramework) {

      LOG.info("Failover: Test framework - disabled");

      return "";

    }

    String fsck = "";

    LOG.info("Failover: Test framework - verification - starting...");

    AvatarFailoverSnapshot snapshot = new AvatarFailoverSnapshot();

    File snapshotFile = getSnapshotFile(confg);

    DataInputStream in = new DataInputStream(

        new BufferedInputStream(new FileInputStream(snapshotFile)));

    try {

      snapshot.readFields(in);

      if (in.readBoolean()) {

        LOG.info("Failover: Test framework - found fsck data");

        fsck = Text.readString(in);

      }

    } finally {

      in.close();

    }

   

    LOG.info("Failover: Test framework - verifying open files: found "

        + snapshot.getOpenFilesInfo().getOpenFiles().size()

        + " files in the test snapshot"); 

    verifyOpenFiles(snapshot.getOpenFilesInfo());

   

    LOG.info("Failover: Test framework - verifying closed files: found "

        + snapshot.getSampledFiles().size()

        + " files in the test snapshot");   

    for (FileStatusExtended stat : snapshot.getSampledFiles()) {

      verifySnapshotSampledFile(stat);

    }

    LOG.info("Failover: Test framework - verification - succeeded");

    return fsck;

  }

 

  protected String runFailoverFsck() throws IOException {

    Map<String, String[]> pmap = new HashMap<String, String[]>();

    pmap.put("path", new String[] {"/"});

   

    // run fsck

    StringWriter stringWriter = new StringWriter();

    NamenodeFsck fscker = new NamenodeFsck(confg, this,

        pmap, new PrintWriter(stringWriter));

    fscker.fsck();

    return stringWriter.toString();

  }

  /**

   * @inheritDoc

   */

  @Override

  public synchronized void setAvatar(Avatar avatar) throws IOException {

    setAvatar(avatar, false);

  }

  /**

   * @inheritDoc

   */

  public synchronized void setAvatar(Avatar avatar, boolean force)

      throws IOException {

    if (avatar == currentAvatar) {

      LOG.info("Failover: Trying to change avatar to " + avatar +

               " but am already in that state.");

      return;

    }

    if (avatar == Avatar.STANDBY) {   // ACTIVE to STANDBY

      String msg = "Failover: Changing state from active to standby is not allowed." +

                   "If you really want to pause your primary, put it in safemode.";

      LOG.warn(msg);

      throw new IOException(msg);

    } else {                  // STANDBY to ACTIVE

      // Check to see if the primary is somehow checkpointing itself. If so, then

      // refuse to switch to active mode. This check is not foolproof but is a

      // defensive mechanism to prevent administrator errors.

      try {

        if (!zkIsEmpty()) {

          throw new IOException("Can't switch the AvatarNode to primary since " +

              "zookeeper record is not clean. Either use shutdownAvatar to kill " +

              "the current primary and clean the ZooKeeper entry, " +

              "or clear out the ZooKeeper entry if the primary is dead");

        }

      } catch (Exception ex) {

        throw new IOException("Cancelling setAvatar because of Exception", ex);

      }

      if (standby.hasStaleCheckpoint()) {

        String msg = "Failover: Failed to change avatar from " + currentAvatar +

                     " to " + avatar +

                     " because the Standby has not yet consumed all transactions.";

        LOG.warn(msg);

        throw new IOException(msg);

      }

      InjectionHandler

          .processEvent(InjectionEvent.AVATARNODE_AFTER_STALE_CHECKPOINT_CHECK);

      ZookeeperTxId zkTxId = null;

      if (!force) {

        zkTxId = getLastTransactionId();

        standby.quiesce(zkTxId.getTransactionId());

      } else {

        standby.quiesce(TXID_IGNORE);

      }

      cleaner.stop();

      cleanerThread.interrupt();

      try {

        cleanerThread.join();

      } catch (InterruptedException iex) {

        Thread.currentThread().interrupt();