Examples of com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.FlumeNode
This is a configurable flume node. It has four main parts: 1) a http server that shows its status, 2) A LivenessManager. This encapsulates heartbeat communications with master that indicate the node is alive, and can trigger config updates. 3) a MasterRPC layer. This encapsulates the RPC comms with the master. It can be replaced later with a different master communication mechanism (Avro or ZK). 4) A LogicalNodeManager. There now can be more than one logical node on a single node. This class manages them. Threads in the status server do not prevent exit. Threads in liveness manager will prevent exit as will any running threads in nodethread manager.

      IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    String nodename = "bar";
    flumeMaster.getSpecMan().addLogicalNode(nodename, "foo");
    FlumeNode n = new FlumeNode(FlumeConfiguration.get(), nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();


    // jumpstart the heartbeat (get foo register, and its default logicalNode
    // started)
    n.getLivenessManager().heartbeatChecks();


    // One for the logical node by default, one for foo
    assertEquals(2, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get("foo").state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    // TODO: uncomment when there is a clean way to get at the reportable
    n.stop();
  }

View Full Code Here

      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    String nodename = "foo";
    FlumeConfiguration conf = FlumeConfiguration.createTestableConfiguration();
    FlumeNode n = new FlumeNode(conf, nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();


    String node2 = "bar";
    FlumeNode n2 = new FlumeNode(conf, node2, new DirectMasterRPC(flumeMaster),
        false, false);
    n2.start();


    String node3 = "baz";
    FlumeNode n3 = new FlumeNode(conf, node3, new DirectMasterRPC(flumeMaster),
        false, false);
    n3.start();


    // jumpstart the heartbeat (get foo register, and its default logicalNode
    // started)
    n.getLivenessManager().heartbeatChecks();
    n2.getLivenessManager().heartbeatChecks();
    n3.getLivenessManager().heartbeatChecks();


    assertEquals(3, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");
    sh
        .executeLine("exec config bar 'synth(50)' '{delay(100) => accumulator(\"count2\") }' ");
    sh
        .executeLine("exec config baz 'synth(75)' '{delay(100) => accumulator(\"count3\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesDone 0 foo bar baz");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get(nodename).state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(100, cnt.getCount());
    AccumulatorSink cnt2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    assertEquals(50, cnt2.getCount());
    AccumulatorSink cnt3 = (AccumulatorSink) ReportManager.get().getReportable(
        "count3");
    assertEquals(75, cnt3.getCount());
    n.stop();
    n2.stop();
    n3.stop();
  }

View Full Code Here

  public void testNodesActive() throws InterruptedException,
      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    String nodename = "foo";
    FlumeNode n = new FlumeNode(FlumeConfiguration.get(), nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();


    // jumpstart the heartbeat (get foo register, and its default logicalNode
    // started)
    n.getLivenessManager().heartbeatChecks();


    assertEquals(1, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    // this will run for 10 seconds
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesActive 0 foo");
    n.getLivenessManager().heartbeatChecks();
    NodeStatus status = flumeMaster.getStatMan().getNodeStatuses()
        .get(nodename);
    NodeState active = NodeState.ACTIVE;
    assertEquals(status.state, active);


    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    status = flumeMaster.getStatMan().getNodeStatuses().get(nodename);
    NodeState idle = NodeState.IDLE;
    assertEquals(status.state, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(100, cnt.getCount());
    n.stop();
  }

View Full Code Here

    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");


    MasterRPC rpc = new DirectMasterRPC(master);


    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());


  }

View Full Code Here

  @Test
  public void testMasterLogicalNodeSpawnedAtNode() throws IOException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(NetUtils.localhost(),
        node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

  public void testUnmappedLogicalNodeGetsDecommissioned() throws IOException,
      InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");


    master.getStatMan().updateHeartbeatStatus(NetUtils.localhost(),
        node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);


    master.getSpecMan().unmapLogicalNode(NetUtils.localhost(), "foo");


    master.getStatMan().checkup();


    assertEquals(NodeState.DECOMMISSIONED, master.getStatMan()
        .getNodeStatuses().get("foo").state);


    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
    master.getStatMan().updateHeartbeatStatus(NetUtils.localhost(),
        node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);


    master.getStatMan().checkup();


    assertEquals(NodeState.ACTIVE, master.getStatMan().getNodeStatuses().get(
        "foo").state);

View Full Code Here

   */
  @Test
  public void testMasterDecomission() throws IOException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    master.getSpecMan().addLogicalNode(NetUtils.localhost(),
        node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

  public void testUnmapLogicalNode() throws IOException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));

View Full Code Here

  public void testDuplicateSpawn() throws IOException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    // these are spawn commands
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    // there should not be duplicates in the mapping table.
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");


    Multimap<String, String> mapping = master.getSpecMan().getLogicalNodeMap();
    assertEquals(3, mapping.size());


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }

View Full Code Here

    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master = new FlumeMaster(new CommandManager(),
        new ConfigManager(), new StatusManager(), new MasterAckManager(), cfg);


    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    String local = NetUtils.localhost();


    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");
    master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");


    LivenessManager liveMan = node.getLivenessManager();
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();


    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.diskfailover.DiskFailoverManager

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverAgent

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.durability.NaiveFileWALManager

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.WALManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.