Examples of com.cloudera.flume.conf.Context

com.cloudera.flume.conf.Context
This context is an abstraction for scopes. Prior to this there were one scopes the physical node (essentially conf file). This allows for new subscopes such as logical node contexts. This allows information to be passed to specific logical nodes (such as logicalnode name). This interface will likely subsume FlumeConfiguration (physical node scope), and configuration information grabbed from the master such as as logical node specific failover chains (master scope). With scope chaining and shadowing we can in the future introduce node specific configuration settings transparently.

   */
  @Test
  public void testBuilderHandleEvilSlashes() throws FlumeSpecException {


    String src4 = "collectorSink(\"file://C:\\tmp\\test\", \"file\", 1000)";
    CollectorSink snk = (CollectorSink) FlumeBuilder.buildSink(new Context(),
        src4);
    assertEquals(
        "escapedCustomDfs(\"file://C:\\tmp\\test\",\"file%{rolltag}\" )",
        snk.roller.getRollSpec());
  }

View Full Code Here

  public void testOpenClose() throws FlumeSpecException, IOException,
      InterruptedException {
    String src2 = "collectorSink(\"file:///tmp/test\",\"testfilename\")";


    for (int i = 0; i < 100; i++) {
      EventSink snk = FlumeBuilder.buildSink(new Context(), src2);
      snk.open();
      snk.close();
    }
  }

View Full Code Here

   * @throws FlumeSpecException
   */
  @Test
  public void testCorrectFilename() throws IOException, InterruptedException,
      FlumeSpecException {
    CollectorSink sink = new CollectorSink(new Context(),
        "file:///tmp/flume-test-correct-filename", "actual-file-", 10000,
        new Tagger() {
          public String getTag() {
            return "tag";
          }

View Full Code Here


    // get collector and deconstruct it so we can control when it rolls (and
    // thus closes hdfs handles).
    String snkspec = "collectorSink(\"file:///" + tmpdir.getAbsolutePath()
        + "\",\"\")";
    CollectorSink coll = (CollectorSink) FlumeBuilder.buildSink(new Context(),
        snkspec);
    RollSink roll = coll.roller;


    // normally inside wal
    NaiveFileWALDeco.AckChecksumRegisterer<EventSink> snk = new NaiveFileWALDeco.AckChecksumRegisterer(

View Full Code Here

   * close can happen before open has completed.
   */
  @Test
  public void testHdfsDownInterruptBeforeOpen() throws FlumeSpecException,
      IOException, InterruptedException {
    final EventSink snk = FlumeBuilder.buildSink(new Context(),
        "collectorSink(\"hdfs://nonexistant/user/foo\", \"foo\")");


    final CountDownLatch done = new CountDownLatch(1);


    Thread t = new Thread("append thread") {

View Full Code Here

   * close always happens after open has completed.
   */
  @Test
  public void testHdfsDownInterruptAfterOpen() throws FlumeSpecException,
      IOException, InterruptedException {
    final EventSink snk = FlumeBuilder.buildSink(new Context(),
        "collectorSink(\"hdfs://nonexistant/user/foo\", \"foo\")");


    final CountDownLatch started = new CountDownLatch(1);
    final CountDownLatch done = new CountDownLatch(1);

View Full Code Here

   */
  @Test
  public void testHdfsDownInterruptAfterOpeningRetry()
      throws FlumeSpecException, IOException, InterruptedException {
    final EventSink snk = new LazyOpenDecorator(FlumeBuilder.buildSink(
        new Context(),
        "collectorSink(\"hdfs://nonexistant/user/foo\", \"foo\")"));


    final CountDownLatch started = new CountDownLatch(1);
    final CountDownLatch done = new CountDownLatch(1);

View Full Code Here

  }


  @Test
  public void testBestEffortBuilder() throws FlumeSpecException {
    String snk = " agentBestEffortSink";
    FlumeBuilder.buildSink(new Context(), snk);


    String snk2 = "agentBestEffortSink(\"localhost\")";
    FlumeBuilder.buildSink(new Context(), snk2);


    String snk3 = "agentBestEffortSink(\"localhost\", 12345)";
    FlumeBuilder.buildSink(new Context(), snk3);
    try {
      String snk4 = "agentBestEffortSink(\"localhost\", 12345, \"fail\")";
      FlumeBuilder.buildSink(new Context(), snk4);
    } catch (Exception e) {
      return;
    }
    Assert.fail("unexpected fall through");
  }

View Full Code Here


            CounterSink cnt1 = new CounterSink("count." + idx);
            AckChecksumChecker<EventSink> chk = new AckChecksumChecker<EventSink>(
                cnt1);
            NaiveFileWALManager wman1 = new NaiveFileWALManager(f1);
            EventSink snk = new NaiveFileWALDeco(new Context(), chk, wman1,
                new TimeTrigger(1000000), new AckListener.Empty(), 100);


            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.

View Full Code Here

      InterruptedException {


    int count = 20;
    String rpt = "foo";
    String snk = "{ ackChecker => [ console, counter(\"" + rpt + "\") ] }";
    Context ctx = new ReportTestingContext();
    EventSink es = FlumeBuilder.buildSink(ctx, snk);
    // Excessively large timeout because I am manually triggering it.
    NaiveFileWALDeco wal = new NaiveFileWALDeco(ctx, es, node.getWalManager(),
        new TimeTrigger(new ProcessTagger(), 100000), node.getAckChecker()
            .getAgentAckQueuer(), 1000);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.Context

com.cloudera.flume.agent.AgentFailChainSink

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.LogicalNodeManager

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.