Package com.cloudera.flume.agent

Examples of com.cloudera.flume.agent.FlumeNode


        conf.getAgentLogsDir()));
    colAck = (colAck != null) ? colAck : new CollectorAckListener(mock);
    liveman = (liveman != null) ? liveman : new LivenessManager(nodesMan, mock,
        walMan);

    node = new FlumeNode(NetUtils.localhost(), mock, nodesMan, walMan, dfMan,
        colAck, liveman);
  }
View Full Code Here


      IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());

    String nodename = "bar";
    flumeMaster.getSpecMan().addLogicalNode(nodename, "foo");
    FlumeNode n = new FlumeNode(FlumeConfiguration.get(), nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();

    // jumpstart the heartbeat (get foo register, and its default logicalNode
    // started)
    n.getLivenessManager().heartbeatChecks();

    // One for the logical node by default, one for foo
    assertEquals(2, flumeMaster.getStatMan().getNodeStatuses().size());

    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");

    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);

    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get("foo").state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    // TODO: uncomment when there is a clean way to get at the reportable
    n.stop();
  }
View Full Code Here

      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());

    String nodename = "foo";
    FlumeConfiguration conf = FlumeConfiguration.createTestableConfiguration();
    FlumeNode n = new FlumeNode(conf, nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();

    String node2 = "bar";
    FlumeNode n2 = new FlumeNode(conf, node2, new DirectMasterRPC(flumeMaster),
        false, false);
    n2.start();

    String node3 = "baz";
    FlumeNode n3 = new FlumeNode(conf, node3, new DirectMasterRPC(flumeMaster),
        false, false);
    n3.start();

    // jumpstart the heartbeat (get foo register, and its default logicalNode
    // started)
    n.getLivenessManager().heartbeatChecks();
    n2.getLivenessManager().heartbeatChecks();
    n3.getLivenessManager().heartbeatChecks();

    assertEquals(3, flumeMaster.getStatMan().getNodeStatuses().size());

    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");
    sh
        .executeLine("exec config bar 'synth(50)' '{delay(100) => accumulator(\"count2\") }' ");
    sh
        .executeLine("exec config baz 'synth(75)' '{delay(100) => accumulator(\"count3\") }' ");

    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);

    sh.executeLine("waitForNodesDone 0 foo bar baz");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get(nodename).state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(
        "count");
    assertEquals(100, cnt.getCount());
    AccumulatorSink cnt2 = (AccumulatorSink) ReportManager.get().getReportable(
        "count2");
    assertEquals(50, cnt2.getCount());
    AccumulatorSink cnt3 = (AccumulatorSink) ReportManager.get().getReportable(
        "count3");
    assertEquals(75, cnt3.getCount());
    n.stop();
    n2.stop();
    n3.stop();
  }
View Full Code Here

        if (argv.length >= 1) {
          delayMillis = Long.parseLong(argv[0]);
        }

        // TODO (jon) this will cause problems with multiple nodes in same JVM
        FlumeNode node = FlumeNode.getInstance();
        String walnode = context.getValue(LogicalNodeContext.C_LOGICAL);
        if (argv.length >= 2) {
          walnode = argv[1];
        }
        Preconditions.checkArgument(walnode != null,
            "Context does not have a logical node name");

        long checkMs = 250; // TODO replace with config var;
        if (argv.length >= 3) {
          checkMs = Long.parseLong(argv[2]);
        }

        // TODO (jon) this is going to be unsafe because it creates before open.
        // This needs to be pushed into the logic of the decorator
        WALManager walman = node.getAddWALManager(walnode);
        return new NaiveFileWALDeco(context, null, walman, new TimeTrigger(
            delayMillis), node.getAckChecker().getAgentAckQueuer(), checkMs);
      }
    };
  }
View Full Code Here

  public void doTestContextConcurrentWALMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    FlumeTestHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);

    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;
      int count = events + i;
      String src = "asciisynth(" + count + ",100)";
      String snk = " { ackedWriteAhead(15000) => {ackChecker => counter(\""
          + report + "\") }}";

      node.getLogicalNodeManager().testingSpawn(name, src, snk);
    }

    // wait for WALs to flush.
    waitForEmptyWALs(master, node, timeout);
View Full Code Here

  public void doTestLogicalNodesConcurrentDFOMans(final int threads,
      final int events, int timeout) throws IOException, InterruptedException,
      FlumeSpecException {
    FlumeTestHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
    final Reportable[] dfos = new Reportable[threads];

    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;
      int count = events + i;
      String src = "asciisynth(" + count + ",100)";
      String snk = "{ diskFailover => counter(\"" + report + "\") } ";
      node.getLogicalNodeManager().testingSpawn(name, src, snk);
      dfos[i] = node.getLogicalNodeManager().get(name);
    }

    // TODO (jon) using sleep is cheating to give all threads a chance to start.
    // Test seems flakey without this due to a race condition.
    Thread.sleep(500);
View Full Code Here

      FlumeSpecException {
    final String lnode = "DFOSimple";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());
    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid dfo host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentDFOSink(\"localhost\", 12345)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();

    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 10000));

    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);

    // as node do heartbeat and update due to decommission
    liveMan.heartbeatChecks();
    LOG.info("!!! node should be decommissioning on node");
    assertTrue("Attempting to decommission driver timed out",
        d.waitForAtLeastState(DriverState.IDLE, 10000));

    assertEquals("Only expected default logical node", 1, node
        .getLogicalNodeManager().getNodes().size());
    assertNull(node.getLogicalNodeManager().get(lnode));
  }
View Full Code Here

      IOException, FlumeSpecException {
    final String lnode = "DFOBadDNS";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());
    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid dfo host");
    master.getSpecMan().setConfig("node2", "flow", "asciisynth(0)",
        "agentDFOSink(\"invalid\", 12346)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();

    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 20000));

    // update config node to something that will be interrupted.
    LOG.info("!!! decommissioning node on master");
    master.getSpecMan().removeLogicalNode(lnode);

    liveMan.heartbeatChecks();
    LOG.info("!!! logical node should be decommissioning on node");
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.IDLE, 20000));
    LOG.info("Clean close.");

    // false means timeout, takes about 10 seconds to shutdown.
    assertTrue("Attempting to decommission driver timed out",
        d.waitForAtLeastState(DriverState.IDLE, 10000));

    assertEquals("Only expected default logical node", 1, node
        .getLogicalNodeManager().getNodes().size());
    assertNull(node.getLogicalNodeManager().get(lnode));
  }
View Full Code Here

      IOException, FlumeSpecException {
    final String lnode = "e2eSimple";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid e2e host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentE2ESink(\"localhost\", 12347)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();

    // TODO It we only wait for opening state, this test can hang
    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 10000));

    // update config node to something that will be interrupted.
View Full Code Here

      IOException, FlumeSpecException {
    final String lnode = "e2eBadDNS";
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    LivenessManager liveMan = node.getLivenessManager();
    // update config node to something that will be interrupted.
    LOG.info("setting to invalid e2e host");
    master.getSpecMan().setConfig(lnode, "flow", "asciisynth(0)",
        "agentE2ESink(\"localhost\", 12348)");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), lnode);
    liveMan.heartbeatChecks();

    // TODO It we only wait for opening state, this test can hang
    LogicalNode n = node.getLogicalNodeManager().get(lnode);
    Driver d = n.getDriver();
    assertTrue("Attempting to start driver timed out",
        d.waitForAtLeastState(DriverState.ACTIVE, 15000));

    // update config node to something that will be interrupted.
View Full Code Here

TOP

Related Classes of com.cloudera.flume.agent.FlumeNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.