Examples of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.master.FlumeMaster
This is a first cut at a server for distributing configurations to different flume client machines. Right now this is a SPOF and not reliable or persistent, but could eventually hooked into BDB or ZK or something to make configurations more reliable.

   * flumebase is running in a "standalone" mode.
   */
  private void startMaster() {
    LOG.info("Starting standalone Flume master.");
    mFlumeConf.set("flume.master.store", "memory");
    FlumeMaster master = new FlumeMaster(mFlumeConf, false);
    try {
      master.serve();
      mFlumeMaster = master;
    } catch (IOException ioe) {
      LOG.error("IOException starting Flume master: " + ioe);
    }
  }

View Full Code Here

        .get()));
    StatusManager statman = new StatusManager();
    flowed = new FlowConfigManager.FailoverFlowConfigManager(cfgMan, statman);
    logical = new LogicalConfigurationManager(flowed, new ConfigManager(),
        statman);
    flumeMaster = new FlumeMaster(new CommandManager(), logical, statman,
        new MasterAckManager(), FlumeConfiguration.get());
  }

View Full Code Here

  void buildMaster() throws IOException {
    cfgMan = new ConfigManager(FlumeMaster.createConfigStore(FlumeConfiguration
        .get()));
    StatusManager statman = new StatusManager();
    flowed = new FlowConfigManager.FailoverFlowConfigManager(cfgMan, statman);
    flumeMaster = new FlumeMaster(new CommandManager(), flowed, statman,
        new MasterAckManager(), FlumeConfiguration.get());
  }

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 0);
        FlumeMaster master = FlumeMaster.getInstance();
        master.getSpecMan().updateAll();
      }
    };
  }

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 0);
        FlumeMaster master = FlumeMaster.getInstance();
        master.getSpecMan().refreshAll();
      }
    };
  }

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 1);
        FlumeMaster master = FlumeMaster.getInstance();
        String node = argv[0];
        master.getStatMan().purge(node);
      }
    };
  }

View Full Code Here

  public static Execable buildExecable() {
    return new Execable() {
      @Override
      public void exec(String[] argv) throws IOException {
        Preconditions.checkArgument(argv.length == 1);
        FlumeMaster master = FlumeMaster.getInstance();
        try {
          master.getSpecMan().setConfig(argv[0],
              FlumeConfiguration.get().getDefaultFlowName(), "null", "null");
        } catch (FlumeSpecException e) {
          throw new IOException(e);
        }
      }

View Full Code Here

   * that node is decomissioned. As part of decomissioning, the act of closing
   * the dfo agent should not hang the node.
   */
  @Test
  public void testActiveDFOClose() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final CountDownLatch done = new CountDownLatch(1);
    new Thread("ActiveDFOClose") {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentDFOSink(\"localhost\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(10000);


          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");


          // as node do heartbeat and update due to decommission
          liveMan.heartbeatChecks();
          LOG.info("!!! node should be decommissioning on node");

View Full Code Here

   * This test differs from the previous by having an bad dns name/request that
   * will eventually fail (ubuntu/java1.6 takes about 10s)
   */
  @Test
  public void testActiveDFOCloseBadDNS() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentDFOSink(\"invalid\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(20000); // Takes 10s for dns to fail


          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");
          liveMan.heartbeatChecks();
          LOG.info("!!! logical node should be decommissioning on node");


        } catch (Exception e) {
          LOG.error("closed caused an error out: " + e.getMessage(), e);

View Full Code Here

   * that node is decommissioned. As part of decommissioning, the act of closing
   * the E2E agent should not hang the node.
   */
  @Test
  public void testActiveE2EClose() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);


    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());


    final CountDownLatch done = new CountDownLatch(1);
    new Thread("TestDiskFailoverAgent") {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentE2ESink(\"localhost\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(10000);


          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");
          liveMan.heartbeatChecks();
          LOG.info("!!! node should be decommissioning on node");


        } catch (Exception e) {
          LOG.error("closed caused an error out: " + e.getMessage(), e);

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.master.FlumeMaster

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.diskfailover.TestDiskFailoverAgent

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.TestEndToEndAckFailureRecovery

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestLivenessManager

com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.master.commands.PurgeAllCommand

com.cloudera.flume.master.commands.PurgeCommand

com.cloudera.flume.master.commands.RefreshAllCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.