Examples of com.cloudera.flume.util.FlumeShell

com.cloudera.flume.util.FlumeShell
The FlumeShell is a command-line-interface for a Flume master. TODO (jon) move to com.cloudera.flume.shell

   */
  @Test
  public void testSetChokeLimit() throws InterruptedException,
      TTransportException, IOException {


    FlumeShell sh = new FlumeShell();


    sh
        .executeLine("connect localhost:"
            + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh.executeLine("exec setChokeLimit physNode choke 786");
    Clock.sleep(250);
    assertEquals(786, flumeMaster.getSpecMan().getChokeMap("physNode").get(
        "choke").intValue());
  }

View Full Code Here

  @Test
  public void testConnectMultiConfig() throws InterruptedException,
      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    FlumeShell sh = new FlumeShell();
    sh
        .executeLine("connect localhost:"
            + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh.executeLine("exec multiconfig 'localhost:text(\"/var/log/messages\") | "
        + "{delay (250) => console(\"avrojson\") };'");
    Clock.sleep(250);
    assertTrue(flumeMaster.getSpecMan().getAllConfigs().size() > 0);
  }

View Full Code Here

   * Create a master, connect via shell, create some logical nodes, save the
   * config for the node and check if the output looks as expected.
   */
  @Test
  public void testSaveConfigCommand() throws IOException {
    FlumeShell sh = new FlumeShell();
    long retval;


    retval = sh.executeLine("connect localhost:"
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    assertEquals(0, retval);


    retval = sh.executeLine("exec config foo 'null' 'console'");
    assertEquals(0, retval);


    File saveFile = File.createTempFile("test-flume", "");
    saveFile.delete();
    saveFile.deleteOnExit();


    retval = sh.executeLine("exec save '" + saveFile.getAbsolutePath() + "'");
    assertEquals(0, retval);


    BufferedReader in = new BufferedReader(new FileReader(saveFile));
    assertEquals("foo : null | console;", in.readLine());
    assertNull(in.readLine());

View Full Code Here

   * Create a master, create a config file, connect via shell, load config file,
   * compare if flow looks as expected in FlumeConfigData.
   */
  @Test
  public void testLoadConfigCommand() throws IOException {
    FlumeShell sh = new FlumeShell();
    long retval;


    retval = sh.executeLine("connect localhost:"
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    assertEquals(0, retval);


    File saveFile = File.createTempFile("test-flume", "");
    saveFile.deleteOnExit();
    BufferedWriter out = new BufferedWriter(new FileWriter(saveFile));
    out.write("foo : null | console;\n");
    out.close();


    retval = sh.executeLine("exec load '" + saveFile.getAbsolutePath() + "'");
    assertEquals(0, retval);


    ConfigurationManager manager = flumeMaster.getSpecMan();
    FlumeConfigData data = manager.getConfig("foo");
    assertEquals(data.getSinkConfig(), "console");

View Full Code Here

   * 
   * @throws InterruptedException
   */
  @Test
  public void testGetMappings() throws InterruptedException {
    FlumeShell sh = new FlumeShell();
    long retval;


    retval = sh.executeLine("connect localhost:"
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    assertEquals(0, retval);


    retval = sh.executeLine("getmappings");
    assertEquals(0, retval);


    assertEquals(0, flumeMaster.getSpecMan().getLogicalNodeMap().size());


    Clock.sleep(1000);


    retval = sh
        .executeLine("exec config foo 'tail(\"/var/log/messages\")' 'console(\"avrojson\")'");
    assertEquals(0, retval);
    retval = sh
        .executeLine("exec config bar 'tail(\"/var/log/messages2\")' 'console(\"avrojson\")'");
    assertEquals(0, retval);


    retval = sh.executeLine("exec spawn localhost foo");
    assertEquals(0, retval);
    assertEquals(1, flumeMaster.getSpecMan().getLogicalNodeMap().size());


    retval = sh.executeLine("getmappings");
    assertEquals(0, retval);


    retval = sh.executeLine("exec spawn localhost bar");
    assertEquals(0, retval);
    assertEquals(2, flumeMaster.getSpecMan().getLogicalNodeMap().size());


    retval = sh.executeLine("getmappings");
    assertEquals(0, retval);


    retval = sh.executeLine("getmappings idonotexist");
    assertEquals(0, retval);


    retval = sh.executeLine("getmappings localhost");
    assertEquals(0, retval);
  }

View Full Code Here

  @Test
  public void testCommitTimeOut() throws InterruptedException,
      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    FlumeShell sh = new FlumeShell();
    sh
        .executeLine("connect localhost:"
            + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    long start = Clock.unixTime();
    // wait for a ridiculous amount of time, and force timeout to happen.
    sh.executeLine("exec noop 20000");


    long delta = Clock.unixTime() - start;
    assertTrue(delta < 20000);
    assertTrue(delta > FlumeShell.CMD_WAIT_TIME_MS);
  }

View Full Code Here

  @Test
  public void testSubmitWait() throws InterruptedException,
      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    FlumeShell sh = new FlumeShell();
    sh
        .executeLine("connect localhost:"
            + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    long start = Clock.unixTime();
    // wait for a ridiculous amount of time, and force timeout to happen.
    sh.executeLine("submit noop 3000");
    long delta = Clock.unixTime() - start;


    assertTrue(delta < 3000);


    // implicit wait.
    sh.executeLine("wait");
    delta = Clock.unixTime() - start;
    assertTrue(delta > 3000);
  }

View Full Code Here

    n.getLivenessManager().heartbeatChecks();


    // One for the logical node by default, one for foo
    assertEquals(2, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get("foo").state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    // TODO: uncomment when there is a clean way to get at the reportable

View Full Code Here

    n2.getLivenessManager().heartbeatChecks();
    n3.getLivenessManager().heartbeatChecks();


    assertEquals(3, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");
    sh
        .executeLine("exec config bar 'synth(50)' '{delay(100) => accumulator(\"count2\") }' ");
    sh
        .executeLine("exec config baz 'synth(75)' '{delay(100) => accumulator(\"count3\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesDone 0 foo bar baz");
    n.getLivenessManager().heartbeatChecks();
    NodeState status = flumeMaster.getStatMan().getNodeStatuses().get(nodename).state;
    NodeState idle = NodeState.IDLE;
    assertEquals(status, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(

View Full Code Here

    // started)
    n.getLivenessManager().heartbeatChecks();


    assertEquals(1, flumeMaster.getStatMan().getNodeStatuses().size());


    FlumeShell sh = new FlumeShell();
    sh.executeLine("connect localhost: "
        + FlumeConfiguration.DEFAULT_ADMIN_PORT);
    // this will run for 10 seconds
    sh
        .executeLine("exec config foo 'synth(100)' '{delay(100) => accumulator(\"count\") }' ");


    FlumeConfigData fcd = flumeMaster.getSpecMan().getConfig("foo");
    assertEquals("{delay(100) => accumulator(\"count\") }", fcd.sinkConfig);
    assertEquals("synth(100)", fcd.sourceConfig);
    assertTrue(0 != fcd.timestamp);


    sh.executeLine("waitForNodesActive 0 foo");
    n.getLivenessManager().heartbeatChecks();
    NodeStatus status = flumeMaster.getStatMan().getNodeStatuses()
        .get(nodename);
    NodeState active = NodeState.ACTIVE;
    assertEquals(status.state, active);


    sh.executeLine("waitForNodesDone 0 foo");
    n.getLivenessManager().heartbeatChecks();
    status = flumeMaster.getStatMan().getNodeStatuses().get(nodename);
    NodeState idle = NodeState.IDLE;
    assertEquals(status.state, idle);
    AccumulatorSink cnt = (AccumulatorSink) ReportManager.get().getReportable(

View Full Code Here

0 1 2 3

TOP

Related Classes of com.cloudera.flume.util.FlumeShell

com.cloudera.flume.master.Command

com.cloudera.flume.master.CommandStatus

com.cloudera.flume.shell.TestFlumeShell

jline.ConsoleReader

org.apache.commons.cli.CommandLine

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.HelpFormatter

org.apache.commons.cli.Options

org.apache.commons.cli.PosixParser

org.apache.thrift.protocol.TBinaryProtocol

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.