Examples of com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.FlumeNode
This is a configurable flume node. It has four main parts: 1) a http server that shows its status, 2) A LivenessManager. This encapsulates heartbeat communications with master that indicate the node is alive, and can trigger config updates. 3) a MasterRPC layer. This encapsulates the RPC comms with the master. It can be replaced later with a different master communication mechanism (Avro or ZK). 4) A LogicalNodeManager. There now can be more than one logical node on a single node. This class manages them. Threads in the status server do not prevent exit. Threads in liveness manager will prevent exit as will any running threads in nodethread manager.

  private void startPhysicalNode() {
    boolean startHttp = false;
    boolean isOneShot = false;


    mPhysicalNodeName = "rtsql-" + mHostName;
    FlumeNode node = new FlumeNode(mPhysicalNodeName, mFlumeConf, startHttp, isOneShot);
    node.start();
    addFlumeNode(node);
  }

View Full Code Here

  public void testUnmappedLogicalNodeGetsDecommissioned() throws IOException,
      InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");


    master.getStatMan().updateHeartbeatStatus(NetUtils.localhost(),
        node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);


    master.getSpecMan().unmapLogicalNode(NetUtils.localhost(), "foo");


    master.getStatMan().checkup();


    assertEquals(NodeState.DECOMMISSIONED, master.getStatMan()
        .getNodeStatuses().get("foo").state);


    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
    master.getStatMan().updateHeartbeatStatus(NetUtils.localhost(),
        node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);


    master.getStatMan().checkup();


    assertEquals(NodeState.ACTIVE, master.getStatMan().getNodeStatuses().get(
        "foo").state);

View Full Code Here

   */
  @Test
  public void testMasterDecomission() throws IOException, InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(NetUtils.localhost(),
        node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

  public void testUnmapLogicalNode() throws IOException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));

View Full Code Here

  public void testDuplicateSpawn() throws IOException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    // these are spawn commands
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    // there should not be duplicates in the mapping table.
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    Multimap<String, String> mapping = master.getSpecMan().getLogicalNodeMap();
    assertEquals(3, mapping.size());


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);


    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");
    master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));

View Full Code Here

    cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
    master = new FlumeMaster(new CommandManager(), new ConfigManager(),
        new StatusManager(), new MasterAckManager(), cfg);
    master.serve();
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(NetUtils.localhost(),
        node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

    master = new FlumeMaster(new CommandManager(), new ConfigManager(),
        new StatusManager(), new MasterAckManager(), cfg);


    master.serve();
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));

View Full Code Here


    master = new FlumeMaster(new CommandManager(), new ConfigManager(),
        new StatusManager(), new MasterAckManager(), cfg);
    master.serve();
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");
    master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));

View Full Code Here

  public void testMasterChokeAddedAtFlumeNode() throws IOException {


    FlumeMaster master = new FlumeMaster(cfg);


    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    master.getSpecMan().addChokeLimit(node.getPhysicalNodeName(), "bar", 786);


    // This method updateChokeLimitMap is the same method call in the
    // checkLogicalNodes() in the LivenessManager
    node.getChokeManager().updateChokeLimitMap(
        (HashMap<String, Integer>) master.getSpecMan().getChokeMap(
            node.getPhysicalNodeName()));


    if (!node.getChokeManager().isChokeId("bar")) {
      LOG.error("TestChokeMapping Failed!!");
      LOG
          .error("The ChokeID 'bar' did not get registered with the ChokeManager!!");
      fail("TestChokeMapping Failed!!");
    }

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.diskfailover.DiskFailoverManager

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverAgent

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.WALManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.