Examples of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.master.FlumeMaster
This is a first cut at a server for distributing configurations to different flume client machines. Right now this is a SPOF and not reliable or persistent, but could eventually hooked into BDB or ZK or something to make configurations more reliable.


  public void doTestLogicalNodesConcurrentDFOMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    FlumeTestHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
    final Reportable[] dfos = new Reportable[threads];


    for (int i = 0; i < threads; i++) {
      String name = "test." + i;

View Full Code Here

      }


    });
    FlumeBuilder.setSinkFactory(sf);


    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final LivenessManager liveMan = node.getLivenessManager();
    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        liveMan.start();
        try {
          // update config node to something that will be interrupted.
          LOG.info("Config 1 heartbeat");
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);


          // update config node to something that will be interrupted.
          LOG.info("Config 2 heartbeat");
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);


          // update config node to something that will be interrupted.
          LOG.info("Config 3 heartbeat");
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);
        } catch (IOException e) {
          return; // fail

View Full Code Here

    FlumeNode node = new FlumeNode(cfg);
    node.start();


    // avoiding gossip ack manager until it shuts down cleanly.
    ConfigStore cfgStore = FlumeMaster.createConfigStore(cfg);
    FlumeMaster fm = new FlumeMaster(new CommandManager(), new ConfigManager(
        cfgStore), new StatusManager(), new MasterAckManager(), cfg);


    assertEquals(0, fm.getKnownNodes().size());
    fm.serve();
    LOG.info("flume master 1 open ");
    while (fm.getKnownNodes().size() == 0) {
      Clock.sleep(1000);
    }
    fm.shutdown();
    LOG.info("flume master 1 closed");


    ConfigStore cfgStore2 = FlumeMaster.createConfigStore(cfg);
    FlumeMaster fm2 = new FlumeMaster(new CommandManager(), new ConfigManager(
        cfgStore2), new StatusManager(), new MasterAckManager(), cfg);
    assertEquals(0, fm2.getKnownNodes().size());
    fm2.serve();
    LOG.info("flume master 2 open ");
    while (fm2.getKnownNodes().size() == 0) {
      Clock.sleep(1000);
    }
    fm2.shutdown();
    LOG.info("flume master 2 closed");


  }

View Full Code Here

    cfg.set(FlumeConfiguration.MASTER_STORE, "memory");


    File tmpdir = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    FlumeMaster master = FlumeMaster.getInstance();
    FlumeNode node = new FlumeNode(cfg, "foo", new DirectMasterRPC(master),
        false, false);


    node.getAddDFOManager("foo").open();
    node.getAddWALManager("foo").open();

View Full Code Here

   */
  @Test
  public void testActiveDFOClose() throws InterruptedException, IOException,
      FlumeSpecException {
    final String lnode = "DFOSimple";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());
    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid dfo host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentDFOSink(\"localhost\", 12345)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();


    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 10000));


    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);


    // as node do heartbeat and update due to decommission
    liveMan.heartbeatChecks();
    LOG.info("!!! node should be decommissioning on node");
    assertTrue("Attempting to decommission driver timed out",

View Full Code Here

   */
  @Test
  public void testActiveDFOCloseBadDNS() throws InterruptedException,
      IOException, FlumeSpecException {
    final String lnode = "DFOBadDNS";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());
    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid dfo host");
    master.getSpecMan().setConfig("node2", "flow", "asciisynth(0)",
        "agentDFOSink(\"invalid\", 12346)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();


    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 20000));


    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);


    liveMan.heartbeatChecks();
    LOG.info("!!! logical node should be decommissioning on node");
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.IDLE, 20000));

View Full Code Here

   */
  @Test
  public void testActiveE2ECloseSimple() throws InterruptedException,
      IOException, FlumeSpecException {
    final String lnode = "e2eSimple";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid e2e host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentE2ESink(\"localhost\", 12347)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();


    // TODO It we only wait for opening state, this test can hang
    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 10000));


    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);
    liveMan.heartbeatChecks();
    assertTrue("Attempting to stop driver timed out",
        d.waitForAtLeastState(DriverState.ERROR, 15000));
  }

View Full Code Here

   */
  @Test
  public void testActiveE2ECloseBadDNS() throws InterruptedException,
      IOException, FlumeSpecException {
    final String lnode = "e2eBadDNS";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid e2e host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentE2ESink(\"localhost\", 12348)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();


    // TODO It we only wait for opening state, this test can hang
    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 15000));


    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);
    liveMan.heartbeatChecks();
    assertTrue("Attempting to stop driver timed out",
        d.waitForAtLeastState(DriverState.ERROR, 15000));
  }

View Full Code Here


  public void doTestContextConcurrentWALMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    BenchmarkHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);


    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;

View Full Code Here

        tmpdir.getAbsolutePath());
    System.out.println("Writing out tempdir: " + tmpdir.getAbsolutePath());


    // This will register the FlumeNode with a MockMasterRPC so it doesn't go
    // across the network
    master = new FlumeMaster(FlumeConfiguration.get());
    mock = new DirectMasterRPC(master);
    node = new FlumeNode(mock, false /* starthttp */, false /* onshot */);


  }

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverAgent

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.TestEndToEndAckFailureRecovery

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestLivenessManager

com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.master.commands.PurgeAllCommand

com.cloudera.flume.master.commands.PurgeCommand

com.cloudera.flume.master.commands.RefreshAllCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.