Examples of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.master.FlumeMaster
This is a first cut at a server for distributing configurations to different flume client machines. Right now this is a SPOF and not reliable or persistent, but could eventually hooked into BDB or ZK or something to make configurations more reliable.

   * This test differs from the previous by having an bad dns name/request that
   * will eventually fail (ubuntu/java1.6 takes about 10s)
   */
  @Test
  public void testActiveE2ECloseBadDNS() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentE2ESink(\"localhost\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(15000); // Takes 10s for dns to fail


          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");
          liveMan.heartbeatChecks();
          LOG.info("!!! node should be decommissioning on node");


        } catch (Exception e) {
          LOG.error("closed caused an error out: " + e.getMessage(), e);

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 0);
        FlumeMaster master = FlumeMaster.getInstance();
        master.getStatMan().purgeAll();
      }
    };
  }

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 1);
        FlumeMaster master = FlumeMaster.getInstance();
        String node = argv[0];
        master.getSpecMan().refresh(node);
      }
    };
  }

View Full Code Here

      }


    });
    FlumeBuilder.setSinkFactory(sf);


    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);


          // update config node to something that will be interrupted.
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);


          // update config node to something that will be interrupted.
          master.getSpecMan().setConfig(NetUtils.localhost(), "flow",
              "asciisynth(0)", "hang");
          liveMan.heartbeatChecks();
          Thread.sleep(250);
        } catch (IOException e) {
          return; // fail

View Full Code Here


  public void doTestContextConcurrentWALMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    BenchmarkHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);


    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;

View Full Code Here

        tmpdir.getAbsolutePath());
    System.out.println("Writing out tempdir: " + tmpdir.getAbsolutePath());


    // This will register the FlumeNode with a MockMasterRPC so it doesn't go
    // across the network
    master = new FlumeMaster(FlumeConfiguration.get());
    mock = new DirectMasterRPC(master);
    node = new FlumeNode(mock, false /* starthttp */, false /* onshot */);


  }

View Full Code Here

    FlumeNode node = new FlumeNode(cfg);
    node.start();


    // avoiding gossip ack manager until it shuts down cleanly.
    ConfigStore cfgStore = FlumeMaster.createConfigStore(cfg);
    FlumeMaster fm = new FlumeMaster(new CommandManager(), new ConfigManager(
        cfgStore), new StatusManager(), new MasterAckManager(), cfg);


    assertEquals(0, fm.getKnownNodes().size());
    fm.serve();
    LOG.info("flume master 1 open ");
    while (fm.getKnownNodes().size() == 0) {
      Clock.sleep(1000);
    }
    fm.shutdown();
    LOG.info("flume master 1 closed");


    ConfigStore cfgStore2 = FlumeMaster.createConfigStore(cfg);
    FlumeMaster fm2 = new FlumeMaster(new CommandManager(), new ConfigManager(
        cfgStore2), new StatusManager(), new MasterAckManager(), cfg);
    assertEquals(0, fm2.getKnownNodes().size());
    fm2.serve();
    LOG.info("flume master 2 open ");
    while (fm2.getKnownNodes().size() == 0) {
      Clock.sleep(1000);
    }
    fm2.shutdown();
    LOG.info("flume master 2 closed");


  }

View Full Code Here

    cfg.set(FlumeConfiguration.MASTER_STORE, "memory");


    File tmpdir = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    FlumeMaster master = FlumeMaster.getInstance();
    FlumeNode node = new FlumeNode(cfg, "foo", new DirectMasterRPC(master),
        false, false);


    node.getAddDFOManager("foo").open();
    node.getAddWALManager("foo").open();

View Full Code Here


  public void doTestLogicalNodesConcurrentDFOMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    BenchmarkHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
    final Reportable[] dfos = new Reportable[threads];


    for (int i = 0; i < threads; i++) {
      String name = "test." + i;

View Full Code Here


  public void doTestContextConcurrentWALMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    FlumeTestHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);


    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverAgent

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.TestEndToEndAckFailureRecovery

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestLivenessManager

com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.master.commands.PurgeAllCommand

com.cloudera.flume.master.commands.PurgeCommand

com.cloudera.flume.master.commands.RefreshAllCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.