Package org.apache.hadoop.hdfs

Examples of org.apache.hadoop.hdfs.AvatarZooKeeperClient

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    long ssid = -1;
    int maxTries = conf.getInt("dfs.avatarnode.sync.ssidtxid.retries", 3);
    // Whether or not to verify the sessionid after writing it to ZK.
    int tries = 0;
    while (tries < maxTries) {
      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(conf, null);
      try {
        ssid = now();
        zk.registerPrimarySsId(getClusterAddress(conf), ssid);
        // Be extra careful and verify the data was synced to zk.
        Long ssIdInZk = zk.getPrimarySsId(getClusterAddress(conf));
        if (ssid != ssIdInZk) {
          throw new IOException("Session Id in the NameNode : " + ssid +
              " does not match the session Id in Zookeeper : " + ssIdInZk);
        }
        break;
      } catch(Exception e) {
        if (tries == maxTries - 1 ) {
          throw new IOException(e);
        }
      } finally {
        try {
          zk.shutdown();
        } catch (InterruptedException ie) {
          if (tries == maxTries - 1) {
            throw new IOException(ie);
          }
        }
View Full Code Here
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
    ZookeeperTxId zkTxid = new ZookeeperTxId(this.sessionId, lastTxid,
        totalBlocks);
    int maxTries = startupConf.getInt("dfs.avatarnode.sync.ssidtxid.retries", 3);
    int tries = 0;
    while (true) {
      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);
      try {
        zk.registerLastTxId(getClusterAddress(this.startupConf), zkTxid);
        return;
      } catch (Exception e) {
        if (tries > maxTries) {
          throw new IOException(e);
        } else {
          tries++;
          LOG.warn("Error syncing last txid to zk, retrying ....", e);
          try {
            Thread.sleep(5000);
          } catch (InterruptedException ie) {
            throw new IOException("writeLastTxidToZookeeper() interrupted", ie);
          }
        }
      } finally {
        try {
          zk.shutdown();
        } catch (InterruptedException ie) {
          throw new IOException(ie);
        }
      }
    }
View Full Code Here
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
    }
  }

  private ZookeeperTxId getLastTransactionId() throws IOException {
    try {
      AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);
      try {
        // Gather session id and transaction id data.
        String address = getClusterAddress(this.startupConf);
        long sessionId = zk.getPrimarySsId(address);
        ZookeeperTxId zkTxId = zk.getPrimaryLastTxId(address);
        if (sessionId != zkTxId.getSessionId()) {
          throw new IOException("Session Id in the ssid node : " + sessionId
              + " does not match the session Id in the txid node : "
              + zkTxId.getSessionId());
        }
        return zkTxId;
      } finally {
        zk.shutdown();
      }
    } catch (Exception e) {
      throw new IOException(e);
    }
  }
View Full Code Here
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
  private void registerAsPrimaryToZK() throws IOException {
    // Register client port address.
    String address = getClusterAddress(startupConf);
    String realAddress = getClusterAddress(confg);
    AvatarZooKeeperClient zk = new AvatarZooKeeperClient(confg, null);
    try {
      zk.registerPrimary(address, realAddress, true);

      // Register dn protocol address
      registerAddressToZK(zk, "dfs.namenode.dn-address");

      // Register http address
      registerAddressToZK(zk, "dfs.http.address");

      // Register rpc address
      registerAddressToZK(zk, AvatarNode.DFS_NAMENODE_RPC_ADDRESS_KEY);
    } finally {
      try {
        zk.shutdown();
      } catch (InterruptedException e) {
        throw new IOException("Could not shutdown zk client", e);
      }
    }
  }
View Full Code Here
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
    String fsname = defaultAddr.getHostName() + ":" + defaultAddr.getPort();
    InetSocketAddress actualAddr = NameNode.getClientProtocolAddress(conf);
    String actualName = actualAddr.getHostName() + ":" + actualAddr.getPort();


    AvatarZooKeeperClient zk = new AvatarZooKeeperClient(conf, null);
    boolean zkRegistryMatch = true;
    boolean primaryPresent = false;
    String errorMsg = null;
    try {
      Stat stat = new Stat();
      String zkRegistry = zk.getPrimaryAvatarAddress(fsname, stat, false);
      if (zkRegistry == null) {
        // The registry is empty. Usually this means failover is in progress
        // we need to manually fix it before starting primary
        errorMsg = "A zNode that indicates the primary is empty. "
            + "AvatarNode can only be started as primary if it "
            + "is registered as primary with ZooKeeper";
        zkRegistryMatch = false;
      } else {
        primaryPresent = true;
        if (!zkRegistry.equalsIgnoreCase(actualName)) {
          zkRegistryMatch = false;
          errorMsg = "Registration information in ZooKeeper doesn't "
              + "match the address of this node. AvatarNode can "
              + "only be started as primary if it is registered as "
              + "primary with ZooKeeper. zkRegistry = " + zkRegistry
              + ", actual name = " + actualName;
        }
      }
      if (!startInfo.isStandby) {
        isPrimaryAlive(zkRegistry);
      }
    } catch (Exception e) {
      LOG.error("Got Exception reading primary node registration "
          + "from ZooKeeper. Aborting the start", e);
      zkRegistryMatch = false;

    } finally {
      try {
        zk.shutdown();
      } catch (InterruptedException e) {
        LOG.error("Error shutting down ZooKeeper client", e);
      }
    }
    if (!zkRegistryMatch && !startInfo.isStandby) {
View Full Code Here
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
 
  private boolean zkIsEmpty() throws Exception {
      InetSocketAddress defaultAddr = NameNode.getClientProtocolAddress(startupConf);
      String fsname = defaultAddr.getHostName() + ":" + defaultAddr.getPort();

      AvatarZooKeeperClient zk =
        new AvatarZooKeeperClient(this.confg, null);
      try {
        Stat stat = new Stat();
        String zkRegistry = zk.getPrimaryAvatarAddress(fsname, stat, false);
        return zkRegistry == null;
      } catch (Exception e) {
        LOG.error("Got Exception reading primary node registration " +
            "from ZooKeeper.", e);
        throw e;
      } finally {
        try {
          zk.shutdown();
        } catch (InterruptedException e) {
          LOG.error("Error shutting down ZooKeeper client", e);
        }
      }
  }
View Full Code Here
300
301
302
303
304
305
306
307
      this.nameAddr2 = nameAddr2;
      this.avatarAddr1 = avatarAddr1;
      this.avatarAddr2 = avatarAddr2;
      this.defaultAddr = defaultAddr.getHostName() + ":" + defaultAddr.getPort();
      this.nameserviceId = nameserviceId;
      zkClient = new AvatarZooKeeperClient(getConf(), null);
      this.nsRegistration = new DatanodeRegistration(getMachineName());
    }
View Full Code Here
91
92
93
94
95
96
97
   * Sets the backing configuration source
   */
  public void setConfSource(Configurable src) {
    validateConfigFile(src.getConf());
    confSrc = src;
    zkClient = new AvatarZooKeeperClient(confSrc.getConf(), null, true);
  }
View Full Code Here
352
353
354
355
356
357
358
359
      this.nameAddr2 = nameAddr2;
      this.avatarAddr1 = avatarAddr1;
      this.avatarAddr2 = avatarAddr2;
      this.defaultAddr = defaultAddr;
      this.nameserviceId = nameserviceId;
      zkClient = new AvatarZooKeeperClient(getConf(), null);
      this.nsRegistration = new DatanodeRegistration(getMachineName());
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
   * @return the session id and last transaction id information from zookeeper
   * @throws IOException
   */
  static ZookeeperTxId checkZooKeeperBeforeFailover(Configuration startupConf,
      Configuration confg, boolean noverification) throws IOException {
    AvatarZooKeeperClient zk = null;
    String fsname = startupConf.get(NameNode.DFS_NAMENODE_RPC_ADDRESS_KEY);
   
    int maxTries = startupConf.getInt("dfs.avatarnode.zk.retries", 3);
    Exception lastException = null;
    for (int i = 0; i < maxTries; i++) {
      try {
        zk = new AvatarZooKeeperClient(confg, null, false);
        LOG.info("Failover: Checking if the primary is empty");
        String zkRegistry = zk.getPrimaryAvatarAddress(fsname, new Stat(),
            false, i > 0);
        if (zkRegistry != null) {
          throw new IOException(
              "Can't switch the AvatarNode to primary since "
                  + "zookeeper record is not clean. Either use shutdownAvatar to kill "
                  + "the current primary and clean the ZooKeeper entry, "
                  + "or clear out the ZooKeeper entry if the primary is dead");
        }

        if (noverification) {
          return null;
        }

        LOG.info("Failover: Obtaining last transaction id from ZK");
        String address = startupConf.get(NameNode.DFS_NAMENODE_RPC_ADDRESS_KEY);
        long sessionId = zk.getPrimarySsId(address, i > 0);
        ZookeeperTxId zkTxId = zk.getPrimaryLastTxId(address, i > 0);
        if (sessionId != zkTxId.getSessionId()) {
          throw new IOException("Session Id in the ssid node : " + sessionId
              + " does not match the session Id in the txid node : "
              + zkTxId.getSessionId());
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.AvatarZooKeeperClient

  • org.apache.hadoop.hdfs.conf.NameSpaceAddressManager
  • org.apache.hadoop.hdfs.server.datanode.AvatarDataNode$ServicePair
  • org.apache.hadoop.hdfs.server.namenode.AvatarNode
  • org.apache.hadoop.hdfs.server.namenode.AvatarNodeZkUtil
  • org.apache.zookeeper.data.ACL
  • org.apache.zookeeper.data.Id
  • org.apache.zookeeper.data.Stat
  • org.apache.zookeeper.ZooKeeper
  • org.apache.hadoop.fs.FileAlreadyExistsException
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.