Examples of org.apache.zookeeper.server.ZKDatabase

org.apache.zookeeper.server.ZKDatabase
This class maintains the in memory database of zookeeper server states that includes the sessions, datatree and the committed logs. It is booted up after reading the logs and snapshots from the disk.


    static void shutdownServerInstance(ServerCnxnFactory factory,
            String hostPort)
    {
        if (factory != null) {
            ZKDatabase zkDb = null;
            {
                ZooKeeperServer zs = getServer(factory);
                if (zs != null) {
                    zkDb = zs.getZKDatabase();
                }
            }
            factory.shutdown();
            try {
                if (zkDb != null) {
                    zkDb.close();
                }
            } catch (IOException ie) {
                LOG.warn("Error closing logs ", ie);
            }
            final int PORT = getPort(hostPort);

View Full Code Here

        String session = Long.toHexString(sessionId);
        LOG.info("Searching for txn of session 0x " + session +
                " on peer " + peerId);
        String peerType = peerId == qb.getLeaderIndex() ? "leader" : "follower";
        QuorumPeer peer = qb.getPeerList().get(peerId);
        ZKDatabase db = peer.getActiveServer().getZKDatabase();
        for (Proposal p : db.getCommittedLog()) {
            Assert.assertFalse("Should not see " +
                               TraceFormatter.op2String(p.request.type) +
                               " request from local session 0x" + session +
                               " on the " + peerType,
                               p.request.sessionId == sessionId);

View Full Code Here

        Assert.assertTrue("waiting for server to shutdown",
                ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
        zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
        zks.startdata();


        ZKDatabase db = zks.getZKDatabase();


        // Set sizeLimit to be very high number, so we can pull all transactions
        // from txnlog
        Iterator<Proposal> itr = db.getProposalsFromTxnLog(zxids[0], 10000000);


        int createCount = 0;
        ArrayList<Long> retrievedZxids = new ArrayList<Long>(MSG_COUNT);


        // Get zxid of create requests
        while (itr.hasNext()) {
            Proposal proposal = itr.next();
            TxnHeader hdr = new TxnHeader();
            Record rec = SerializeUtils.deserializeTxn(
                    proposal.packet.getData(), hdr);
            if (hdr.getType() == OpCode.create) {
                retrievedZxids.add(hdr.getZxid());
                createCount++;
            }
        }


        // All zxid should match what we created
        Assert.assertTrue("Zxids missmatches",
                Arrays.equals(zxids, retrievedZxids.toArray(new Long[0])));


        // There should be 2000 create requests
        Assert.assertTrue("create proposal count == " + MSG_COUNT,
                (createCount == MSG_COUNT));


        // We are requesting half the number of transaction from the snapshot
        // this should exceed threshold (ZKDatabase.snapshotSizeFactor)
        db.setSnapshotSizeFactor(0.33);
        long sizeLimit = db.calculateTxnLogSizeLimit();


        itr = db.getProposalsFromTxnLog(zxids[MSG_COUNT / 2], sizeLimit);
        Assert.assertFalse("Expect empty proposal", (itr.hasNext()));
        f.shutdown();
        zks.shutdown();
    }

View Full Code Here

                sk.interestOps(nioSrvCnx.getInterestOps());
                return true;
            }
        });


        ZKDatabase database = new ZKDatabase(null);
        database.setlastProcessedZxid(2L);
        QuorumPeer quorumPeer = mock(QuorumPeer.class);
        FileTxnSnapLog logfactory = mock(FileTxnSnapLog.class);
        // Directories are not used but we need it to avoid NPE
        when(logfactory.getDataDir()).thenReturn(new File(""));
        when(logfactory.getSnapDir()).thenReturn(new File(""));
        FollowerZooKeeperServer fzks = null;


        try {
            // Create a new follower
            fzks = new FollowerZooKeeperServer(logfactory, quorumPeer, database);
            fzks.startup();
            fzks.setServerCnxnFactory(serverCnxnFactory);
            quorumPeer.follower = new MyFollower(quorumPeer, fzks);
            LOG.info("Follower created");
            // Simulate a socket channel between a client and a follower
            final SocketChannel socketChannel = createClientSocketChannel();
            // Create the NIOServerCnxn that will handle the client requests
            final MockNIOServerCnxn nioCnxn = new MockNIOServerCnxn(fzks,
                    socketChannel, sk, serverCnxnFactory, selectorThread);
            sk.attach(nioCnxn);
            // Send the connection request as a client do
            nioCnxn.doIO(sk);
            LOG.info("Client connection sent");
            // Send the valid or invalid session packet to the follower
            QuorumPacket qp = createValidateSessionPacketResponse(!sessionTimedout);
            quorumPeer.follower.processPacket(qp);
            LOG.info("Session validation sent");
            // OK, now the follower knows that the session is valid or invalid, let's try
            // to send the watches
            nioCnxn.doIO(sk);
            // wait for the the request processor to do his job
            Thread.sleep(1000L);
            LOG.info("Watches processed");
            // If session has not been validated, there must be NO watches
            int watchCount = database.getDataTree().getWatchCount();
            if (sessionTimedout) {
                // Session has not been re-validated !
                LOG.info("session is not valid, watches = {}", watchCount);
                assertEquals("Session is not valid so there should be no watches", 0, watchCount);
            } else {

View Full Code Here

    static class SimpleLearnerZooKeeperServer extends LearnerZooKeeperServer {


        Learner learner;


        public SimpleLearnerZooKeeperServer(FileTxnSnapLog ftsl, QuorumPeer self) throws IOException {
            super(ftsl, 2000, 2000, 2000, new ZKDatabase(ftsl), self);
        }

View Full Code Here

        LeadThread leadThread = null;
        Leader leader = null;
        try {              
            // Setup a database with two znodes
            FileTxnSnapLog snapLog = new FileTxnSnapLog(tmpDir, tmpDir);
            ZKDatabase zkDb = new ZKDatabase(snapLog);
            
            Assert.assertTrue(ops >= 1);
            long zxid = ZxidUtils.makeZxid(1, 0);            
            for(int i = 1; i <= ops; i++){
                zxid = ZxidUtils.makeZxid(1, i);
                String path = "/foo-"+ i;
                zkDb.processTxn(new TxnHeader(13,1000+i,zxid,30+i,ZooDefs.OpCode.create), 
                                                new CreateTxn(path, "fpjwasalsohere".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 1));
                Stat stat = new Stat();
                Assert.assertEquals("fpjwasalsohere", new String(zkDb.getData(path, stat, null)));
            }                
            Assert.assertTrue(zxid > ZxidUtils.makeZxid(1, 0));
            
            // Generate snapshot and close files.
            snapLog.save(zkDb.getDataTree(), zkDb.getSessionWithTimeOuts());
            snapLog.close();
            
            QuorumPeer peer = createQuorumPeer(tmpDir);
                        
            leader = createLeader(tmpDir, peer);

View Full Code Here

                try {
                    Assert.assertEquals(0, f.self.getAcceptedEpoch());
                    Assert.assertEquals(0, f.self.getCurrentEpoch());


                    // Setup a database with a single /foo node
                    ZKDatabase zkDb = new ZKDatabase(new FileTxnSnapLog(tmpDir, tmpDir));
                    final long firstZxid = ZxidUtils.makeZxid(1, 1);
                    zkDb.processTxn(new TxnHeader(13, 1313, firstZxid, 33, ZooDefs.OpCode.create), new CreateTxn("/foo", "data1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 1));
                    Stat stat = new Stat();
                    Assert.assertEquals("data1", new String(zkDb.getData("/foo", stat, null)));


                    QuorumPacket qp = new QuorumPacket();
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.FOLLOWERINFO, qp.getType());
                    Assert.assertEquals(qp.getZxid(), 0);
                    LearnerInfo learnInfo = new LearnerInfo();
                    ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo);
                    Assert.assertEquals(learnInfo.getProtocolVersion(), 0x10000);
                    Assert.assertEquals(learnInfo.getServerid(), 0);
                
                    // We are simulating an established leader, so the epoch is 1
                    qp.setType(Leader.LEADERINFO);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    byte protoBytes[] = new byte[4];
                    ByteBuffer.wrap(protoBytes).putInt(0x10000);
                    qp.setData(protoBytes);
                    oa.writeRecord(qp, null);
                
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACKEPOCH, qp.getType());
                    Assert.assertEquals(0, qp.getZxid());
                    Assert.assertEquals(ZxidUtils.makeZxid(0, 0), ByteBuffer.wrap(qp.getData()).getInt());
                    Assert.assertEquals(1, f.self.getAcceptedEpoch());
                    Assert.assertEquals(0, f.self.getCurrentEpoch());
                    
                    // Send the snapshot we created earlier
                    qp.setType(Leader.SNAP);
                    qp.setData(new byte[0]);
                    qp.setZxid(zkDb.getDataTreeLastProcessedZxid());
                    oa.writeRecord(qp, null);
                    zkDb.serializeSnapshot(oa);
                    oa.writeString("BenWasHere", null);
                    qp.setType(Leader.NEWLEADER);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    oa.writeRecord(qp, null);


                    // Get the ack of the new leader
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());
                    Assert.assertEquals(1, f.self.getAcceptedEpoch());
                    Assert.assertEquals(1, f.self.getCurrentEpoch());
                    
                    Assert.assertEquals(firstZxid, f.fzk.getLastProcessedZxid());
                    
                    // Make sure the data was recorded in the filesystem ok
                    ZKDatabase zkDb2 = new ZKDatabase(new FileTxnSnapLog(logDir, snapDir));
                    long lastZxid = zkDb2.loadDataBase();
                    Assert.assertEquals("data1", new String(zkDb2.getData("/foo", stat, null)));
                    Assert.assertEquals(firstZxid, lastZxid);


                    // Propose an update
                    long proposalZxid = ZxidUtils.makeZxid(1, 1000);
                    proposeSetData(qp, proposalZxid, "data2", 2);
                    oa.writeRecord(qp, null);
                    
                    TrackerWatcher watcher = new TrackerWatcher();
                    
                    // The change should not have happened yet, since we haven't committed
                    Assert.assertEquals("data1", new String(f.fzk.getZKDatabase().getData("/foo", stat, watcher)));
                    
                    // The change should happen now
                    qp.setType(Leader.COMMIT);
                    qp.setZxid(proposalZxid);
                    oa.writeRecord(qp, null);
                    
                    qp.setType(Leader.UPTODATE);
                    qp.setZxid(0);
                    oa.writeRecord(qp, null);
                    
                    // Read the uptodate ack
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());
                    
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(proposalZxid, qp.getZxid());
                    
                    watcher.waitForChange();
                    Assert.assertEquals("data2", new String(f.fzk.getZKDatabase().getData("/foo", stat, null)));
                    
                    // check and make sure the change is persisted
                    zkDb2 = new ZKDatabase(new FileTxnSnapLog(logDir, snapDir));
                    lastZxid = zkDb2.loadDataBase();
                    Assert.assertEquals("data2", new String(zkDb2.getData("/foo", stat, null)));
                    Assert.assertEquals(proposalZxid, lastZxid);
                } finally {
                    recursiveDelete(tmpDir);
                }

View Full Code Here

                try {
                    Assert.assertEquals(0, f.self.getAcceptedEpoch());
                    Assert.assertEquals(0, f.self.getCurrentEpoch());


                    // Setup a database with a single /foo node
                    ZKDatabase zkDb = new ZKDatabase(new FileTxnSnapLog(tmpDir, tmpDir));
                    final long firstZxid = ZxidUtils.makeZxid(1, 1);
                    zkDb.processTxn(new TxnHeader(13, 1313, firstZxid, 33, ZooDefs.OpCode.create), new CreateTxn("/foo", "data1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 1));
                    Stat stat = new Stat();
                    Assert.assertEquals("data1", new String(zkDb.getData("/foo", stat, null)));


                    QuorumPacket qp = new QuorumPacket();
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.FOLLOWERINFO, qp.getType());
                    Assert.assertEquals(qp.getZxid(), 0);
                    LearnerInfo learnInfo = new LearnerInfo();
                    ByteBufferInputStream.byteBuffer2Record(ByteBuffer.wrap(qp.getData()), learnInfo);
                    Assert.assertEquals(learnInfo.getProtocolVersion(), 0x10000);
                    Assert.assertEquals(learnInfo.getServerid(), 0);
                
                    // We are simulating an established leader, so the epoch is 1
                    qp.setType(Leader.LEADERINFO);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    byte protoBytes[] = new byte[4];
                    ByteBuffer.wrap(protoBytes).putInt(0x10000);
                    qp.setData(protoBytes);
                    oa.writeRecord(qp, null);
                
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACKEPOCH, qp.getType());
                    Assert.assertEquals(0, qp.getZxid());
                    Assert.assertEquals(ZxidUtils.makeZxid(0, 0), ByteBuffer.wrap(qp.getData()).getInt());
                    Assert.assertEquals(1, f.self.getAcceptedEpoch());
                    Assert.assertEquals(0, f.self.getCurrentEpoch());
                    
                    // Send a diff
                    qp.setType(Leader.DIFF);
                    qp.setData(new byte[0]);
                    qp.setZxid(zkDb.getDataTreeLastProcessedZxid());
                    oa.writeRecord(qp, null);
                    final long createSessionZxid = ZxidUtils.makeZxid(1, 2);
                    proposeNewSession(qp, createSessionZxid, 0x333);
                    oa.writeRecord(qp, null);
                    qp.setType(Leader.COMMIT);
                    qp.setZxid(createSessionZxid);
                    oa.writeRecord(qp, null);
                    qp.setType(Leader.NEWLEADER);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    qp.setData(null);
                    oa.writeRecord(qp, null);
                    qp.setType(Leader.UPTODATE);
                    qp.setZxid(0);
                    oa.writeRecord(qp, null);
                    
                    // Read the uptodate ack
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());
                    
                  
                    // Get the ack of the new leader
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());
                    Assert.assertEquals(1, f.self.getAcceptedEpoch());
                    Assert.assertEquals(1, f.self.getCurrentEpoch());
                    
                    Assert.assertEquals(createSessionZxid, f.fzk.getLastProcessedZxid());
                    
                    // Make sure the data was recorded in the filesystem ok
                    ZKDatabase zkDb2 = new ZKDatabase(new FileTxnSnapLog(logDir, snapDir));
                    zkDb2.loadDataBase();
                    LOG.info("zkdb2 sessions:" + zkDb2.getSessions());
                    Assert.assertNotNull(zkDb2.getSessionWithTimeOuts().get(4L));
                } finally {
                    recursiveDelete(tmpDir);
                }
                
            }

View Full Code Here

        });
    }


    private void deserializeSnapshot(InputArchive ia)
            throws IOException {
        ZKDatabase zkdb = new ZKDatabase(null);
        zkdb.deserializeSnapshot(ia);
        String signature = ia.readString("signature");
        assertEquals("BenWasHere", signature);
    }

View Full Code Here

                try {
                    Assert.assertEquals(0, o.self.getAcceptedEpoch());
                    Assert.assertEquals(0, o.self.getCurrentEpoch());


                    // Setup a database with a single /foo node
                    ZKDatabase zkDb = new ZKDatabase(new FileTxnSnapLog(tmpDir, tmpDir));
                    final long foo1Zxid = ZxidUtils.makeZxid(1, 1);
                    final long foo2Zxid = ZxidUtils.makeZxid(1, 2);
                    zkDb.processTxn(new TxnHeader(13, 1313, foo1Zxid, 33,
                            ZooDefs.OpCode.create), new CreateTxn("/foo1",
                            "data1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
                            false, 1));
                    zkDb.processTxn(new TxnHeader(13, 1313, foo2Zxid, 33,
                            ZooDefs.OpCode.create), new CreateTxn("/foo2",
                            "data1".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
                            false, 1));
                    Stat stat = new Stat();
                    Assert.assertEquals("data1",
                            new String(zkDb.getData("/foo1", stat, null)));
                    Assert.assertEquals("data1",
                            new String(zkDb.getData("/foo2", stat, null)));


                    QuorumPacket qp = new QuorumPacket();
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.OBSERVERINFO, qp.getType());
                    Assert.assertEquals(qp.getZxid(), 0);
                    LearnerInfo learnInfo = new LearnerInfo();
                    ByteBufferInputStream.byteBuffer2Record(
                            ByteBuffer.wrap(qp.getData()), learnInfo);
                    Assert.assertEquals(learnInfo.getProtocolVersion(), 0x10000);
                    Assert.assertEquals(learnInfo.getServerid(), 0);


                    // We are simulating an established leader, so the epoch is 1
                    qp.setType(Leader.LEADERINFO);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    byte protoBytes[] = new byte[4];
                    ByteBuffer.wrap(protoBytes).putInt(0x10000);
                    qp.setData(protoBytes);
                    oa.writeRecord(qp, null);


                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACKEPOCH, qp.getType());
                    Assert.assertEquals(0, qp.getZxid());
                    Assert.assertEquals(ZxidUtils.makeZxid(0, 0), ByteBuffer
                            .wrap(qp.getData()).getInt());
                    Assert.assertEquals(1, o.self.getAcceptedEpoch());
                    Assert.assertEquals(0, o.self.getCurrentEpoch());


                    // Send the snapshot we created earlier
                    qp.setType(Leader.SNAP);
                    qp.setData(new byte[0]);
                    qp.setZxid(zkDb.getDataTreeLastProcessedZxid());
                    oa.writeRecord(qp, null);
                    zkDb.serializeSnapshot(oa);
                    oa.writeString("BenWasHere", null);
                    qp.setType(Leader.NEWLEADER);
                    qp.setZxid(ZxidUtils.makeZxid(1, 0));
                    oa.writeRecord(qp, null);


                    // Get the ack of the new leader
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());
                    Assert.assertEquals(1, o.self.getAcceptedEpoch());
                    Assert.assertEquals(1, o.self.getCurrentEpoch());


                    Assert.assertEquals(foo2Zxid, o.zk.getLastProcessedZxid());


                    // Make sure the data was recorded in the filesystem ok
                    ZKDatabase zkDb2 = new ZKDatabase(new FileTxnSnapLog(
                            logDir, snapDir));
                    long lastZxid = zkDb2.loadDataBase();
                    Assert.assertEquals("data1",
                            new String(zkDb2.getData("/foo1", stat, null)));
                    Assert.assertEquals(foo2Zxid, lastZxid);


                    // Register watch
                    TrackerWatcher watcher = new TrackerWatcher();
                    Assert.assertEquals("data1", new String(o.zk
                            .getZKDatabase().getData("/foo2", stat, watcher)));


                    // Propose /foo1 update
                    long proposalZxid = ZxidUtils.makeZxid(1, 1000);
                    proposeSetData(qp, "/foo1", proposalZxid, "data2", 2);
                    oa.writeRecord(qp, null);


                    // Commit /foo1 update
                    qp.setType(Leader.COMMIT);
                    qp.setZxid(proposalZxid);
                    oa.writeRecord(qp, null);


                    // Inform /foo2 update
                    long informZxid = ZxidUtils.makeZxid(1, 1001);
                    proposeSetData(qp, "/foo2", informZxid, "data2", 2);
                    qp.setType(Leader.INFORM);
                    oa.writeRecord(qp, null);


                    qp.setType(Leader.UPTODATE);
                    qp.setZxid(0);
                    oa.writeRecord(qp, null);


                    // Read the uptodate ack
                    readPacketSkippingPing(ia, qp);
                    Assert.assertEquals(Leader.ACK, qp.getType());
                    Assert.assertEquals(ZxidUtils.makeZxid(1, 0), qp.getZxid());


                    // Data should get updated
                    watcher.waitForChange();
                    Assert.assertEquals("data2", new String(o.zk
                            .getZKDatabase().getData("/foo1", stat, null)));
                    Assert.assertEquals("data2", new String(o.zk
                            .getZKDatabase().getData("/foo2", stat, null)));


                    // Shutdown sequence guarantee that all pending requests
                    // in sync request processor get flush to disk
                    o.zk.shutdown();


                    zkDb2 = new ZKDatabase(new FileTxnSnapLog(logDir, snapDir));
                    lastZxid = zkDb2.loadDataBase();
                    Assert.assertEquals("data2", new String(zkDb2.getData("/foo1", stat, null)));
                    Assert.assertEquals("data2", new String(zkDb2.getData("/foo2", stat, null)));
                    Assert.assertEquals(informZxid, lastZxid);
                } finally {
                    recursiveDelete(tmpDir);
                }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.zookeeper.server.ZKDatabase

com.linkedin.d2.quorum.ZKPeer

io.fabric8.zookeeper.bootstrap.ZooKeeperServerFactory

org.apache.curator.test.TestingZooKeeperMain

org.apache.hadoop.ha.ClientBaseWithFixes

org.apache.hadoop.yarn.lib.TestZKClient

org.apache.jute.BinaryOutputArchive

org.apache.zookeeper.server.admin.Commands$MonitorCommand

org.apache.zookeeper.server.persistence.FileTxnSnapLog.PlayBackListener

org.apache.zookeeper.server.persistence.TxnLog.TxnIterator

org.apache.zookeeper.server.quorum.Leader.Proposal

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.