Package com.cloudera.flume.agent

Examples of com.cloudera.flume.agent.FlumeNode


    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");

    MasterRPC rpc = new DirectMasterRPC(master);

    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());

  }
View Full Code Here


  public void testMasterLogicalNodeSpawnedAtNode() throws IOException,
      InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    master.getSpecMan().addLogicalNode(NetUtils.localhost(),
        node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }
View Full Code Here

        if (argv.length >= 1) {
          delayMillis = Long.parseLong(argv[0]);
        }

        // TODO (jon) this will cause problems with multiple nodes in same JVM
        FlumeNode node = FlumeNode.getInstance();
        String walnode = context.getValue(LogicalNodeContext.C_LOGICAL);
        if (argv.length >= 2) {
          walnode = argv[1];
        }
        Preconditions.checkArgument(walnode != null,
            "Context does not have a logical node name");

        long checkMs = 250; // TODO replace with config var;
        if (argv.length >= 3) {
          checkMs = Long.parseLong(argv[2]);
        }

        // TODO (jon) this is going to be unsafe because it creates before open.
        // This needs to be pushed into the logic of the decorator
        WALManager walman = node.getAddWALManager(walnode);
        return new NaiveFileWALDeco(context, null, walman, new TimeTrigger(
            delayMillis), node.getAckChecker().getAgentAckQueuer(), checkMs);
      }
    };
  }
View Full Code Here

  @Test
  public void testActiveDFOClose() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    final CountDownLatch done = new CountDownLatch(1);
    new Thread("ActiveDFOClose") {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentDFOSink(\"localhost\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(10000);

          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");

          // as node do heartbeat and update due to decommission
          liveMan.heartbeatChecks();
          LOG.info("!!! node should be decommissioning on node");

        } catch (Exception e) {
          LOG.info("closed caused an error out: " + e.getMessage(), e);
          // Right now it takes about 10 seconds for the dfo deco to error out.
          done.countDown();
          return; // expected fail on purpose.
        }

        LOG.info("Clean close.");
        done.countDown();
      }
    }.start();

    // false means timeout, takes about 10 seconds to shutdown.
    assertTrue("close call hung the heartbeat", done
        .await(45, TimeUnit.SECONDS));
    assertEquals(1, node.getLogicalNodeManager().getNodes().size());

  }
View Full Code Here

  @Test
  public void testActiveDFOCloseBadDNS() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentDFOSink(\"invalid\", 12345)");
          master.getSpecMan().addLogicalNode(NetUtils.localhost(), "node1");
          liveMan.heartbeatChecks();
          Thread.sleep(20000); // Takes 10s for dns to fail

          // update config node to something that will be interrupted.
          LOG.info("!!! decommissioning node on master");
          master.getSpecMan().removeLogicalNode("node1");
          liveMan.heartbeatChecks();
          LOG.info("!!! logical node should be decommissioning on node");

        } catch (Exception e) {
          LOG.error("closed caused an error out: " + e.getMessage(), e);
          // Right now it takes about 10 seconds for the dfo deco to error out.
          done.countDown();
          return; // fail
        }

        LOG.info("Clean close.");
        done.countDown();

      }
    }.start();

    // false means timeout, takes about 10 seconds to shutdown.
    assertTrue("close call hung the heartbeat", done
        .await(45, TimeUnit.SECONDS));
    assertEquals(1, node.getLogicalNodeManager().getNodes().size());

  }
View Full Code Here

  @Test
  public void testActiveE2EClose() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    final CountDownLatch done = new CountDownLatch(1);
    new Thread("TestDiskFailoverAgent") {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentE2ESink(\"localhost\", 12345)");
View Full Code Here

  @Test
  public void testActiveE2ECloseBadDNS() throws InterruptedException {
    final FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    final FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    final CountDownLatch done = new CountDownLatch(1);
    new Thread() {
      public void run() {
        LivenessManager liveMan = node.getLivenessManager();
        try {
          // update config node to something that will be interrupted.
          LOG.info("setting to invalid dfo host");
          master.getSpecMan().setConfig("node1", "flow", "asciisynth(0)",
              "agentE2ESink(\"localhost\", 12345)");
View Full Code Here

    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.clear(); // reset all back to defaults.
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());

    mock = new MockMasterRPC();
    node = new FlumeNode(mock, false /* starthttp */, false /* oneshot */);
    ReportManager.get().clear();
  }
View Full Code Here

        conf.getAgentLogsDir()));
    colAck = (colAck != null) ? colAck : new CollectorAckListener(mock);
    liveman = (liveman != null) ? liveman : new LivenessManager(nodesMan, mock,
        walMan);

    node = new FlumeNode(NetUtils.localhost(), mock, nodesMan, walMan, dfMan,
        colAck, liveman);
  }
View Full Code Here

    // parts. Normally this manager would want to delete a wal file (or wal
    // entries). This stubs that out to a call doesn't cause a file not found
    // exception.
    WALManager mockWalMan = mock(WALManager.class);
    BenchmarkHarness.setupFlumeNode(null, mockWalMan, null, null, null);
    FlumeNode node = FlumeNode.getInstance();
    File tmpdir = FileUtil.mktempdir();

    EventSource ackedmem = setupAckRoll();
    Pair<RollSink, EventSink> p = setupSink(node, tmpdir);
    EventSink snk = p.getRight();
    RollSink roll = p.getLeft();
    snk.open();

    String tag1 = roll.getCurrentTag();
    LOG.info(tag1);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    // don't rotate the first one.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // still one ack pending.
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());

    String tag2 = roll.getCurrentTag();
    LOG.info(tag2);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // two acks pending.
    assertEquals(2, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    String tag3 = roll.getCurrentTag();
    LOG.info(tag3);
    snk.append(ackedmem.next()); // ack beg
    snk.append(ackedmem.next()); // data
    snk.append(ackedmem.next()); // ack end
    Clock.sleep(10); // have to make sure it is not in the same millisecond
    roll.rotate();
    // one ack pending
    assertEquals(1, node.getAckChecker().getPendingAckTags().size());
    node.getAckChecker().checkAcks();
    // no more acks pending.
    assertEquals(0, node.getAckChecker().getPendingAckTags().size());

    snk.close();

    FileUtil.rmr(tmpdir);
    BenchmarkHarness.cleanupLocalWriteDir();
View Full Code Here

TOP

Related Classes of com.cloudera.flume.agent.FlumeNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.