Examples of com.cloudera.flume.core.connector.DirectDriver$PumperThread

Package com.cloudera.flume.core.connector

Examples of com.cloudera.flume.core.connector.DirectDriver$PumperThread

com.cloudera.flume.core.connector.DirectDriver
This connector hooks a source to a sink and allow this connection to be stopped and started. This assumes that sources and sinks are opened before the start() method is called


    // one driver for each choke
    DirectDriver[] directDriverArray = new DirectDriver[numDrivers];
    for (int i = 0; i < numDrivers; i++) {
      // Driver i is mapped to ith choke, simple 1 to 1 mapping.
      directDriverArray[i] = new DirectDriver("TestDriver" + i,
          new SynthSourceRndSize(0, minMsgSize, maxMsgSize), tchokeArray[i]);
    }


    // check if all the ChokeIDs are present in the chokeMap
    LOG.info("Running the Individual Test Now!");

View Full Code Here

    // Each driver is randomly assigned to a random choke in the range
    // [0,numChokes)
    int randChokeIndex = 0;
    for (int i = 0; i < numDrivers; i++) {
      randChokeIndex = rand.nextInt(numChokes);
      directDriverArray[i] = new DirectDriver(new SynthSourceRndSize(0,
          minMsgSize, maxMsgSize), tchokeArray[randChokeIndex]);


      // adds this choke to the set of chokesUsed
      chokesUsed.add(tchokeArray[randChokeIndex]);
    }

View Full Code Here

    EventSink snk = new ConsoleEventSink();
    EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
    EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);
    EventSink snk4 = new InMemoryDecorator<EventSink>(snk3);


    DirectDriver connect = new DirectDriver(src, snk4);
    connect.start();
    assertTrue(connect.join(5000));
    snk2.getMetrics().toText(new OutputStreamWriter(System.err));
  }

View Full Code Here

    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    EventSink snk = new ConsoleEventSink();
    EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
    EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);


    DirectDriver connect = new DirectDriver(src, snk3);
    connect.start();
    assertTrue(connect.join(5000));
    snk2.getMetrics().toText(new OutputStreamWriter(System.err));
  }

View Full Code Here

    snk.open();


    EventSource src = FlumeBuilder.buildSource(ctx, SOURCE);
    src.open();


    DirectDriver conn = new DirectDriver(src, snk);
    conn.start();


    conn.join(Long.MAX_VALUE);


    snk.close();
    src.close();
    assertNull(conn.getException());
  }

View Full Code Here

    snk.open();


    EventSource src = srcfact.createSource(ctx, "asciisynth", 25, 100);
    src.open();


    DirectDriver conn = new DirectDriver(src, snk);
    conn.start();


    conn.join(Long.MAX_VALUE);


    snk.close();
    src.close();
  }

View Full Code Here

    EventSink rpcSink = fact.createSink(ctx, "rpcSink", "0.0.0.0", 31337);


    EventSink counter = fact.createSink(ctx, "counter", (Object) "count");
    EventSource txtsrc = srcfact.createSource(ctx, "asciisynth", 25, 100);


    DirectDriver svrconn = new DirectDriver(rpcSrc, counter);
    svrconn.start();
    assertTrue("rpc server took too long to start",
        svrconn.waitForAtLeastState(DriverState.ACTIVE, 1000));


    // start and send the data
    DirectDriver cliconn = new DirectDriver(txtsrc, rpcSink);
    cliconn.start();
    // the avro version sometimes takes a while to start jetty
    assertTrue("rpc client took too long to connect",
        cliconn.waitForAtLeastState(DriverState.ACTIVE, 10000));
    assertTrue("rpc client took too long to close cleanly",
        cliconn.waitForAtLeastState(DriverState.IDLE, 2500));
    Clock.sleep(2000); // data could be stuck in tcp buffer


    svrconn.stop();
    rpcSrc.close(); // force rpc close to make sure next test does not port
                    // conflict.


    assertTrue("rpc server took too long to close cleanly",
        svrconn.waitForAtLeastState(DriverState.IDLE, 2000));


    LOG.info("read " + ((CounterSink) counter).getCount() + " lines");
    assertEquals(LINES, ((CounterSink) counter).getCount());
    assertNull(cliconn.getException());
    assertNull(svrconn.getException());
  }

View Full Code Here

    dfoProducer = dfoMan.getEventSink(ctx, trigger);
    dfoConsumer = dfoMan.getEventSource();


    dfoProducer.open();


    dfoConsumerDriver = new DirectDriver("FileFailover", dfoConsumer, sink);


    dfoConsumerDriver.start();
    boolean success = dfoConsumerDriver.waitForAtLeastState(
        DriverState.OPENING, 1000);
    if (!success) {

View Full Code Here

    snk.open();


    EventSource src = srcfact.getSource("asciisynth", "25", "100");
    src.open();


    DirectDriver conn = new DirectDriver(src, snk);
    conn.start();


    conn.join(Long.MAX_VALUE);


    snk.close();
    src.close();
  }

View Full Code Here

    EventSink counter = fact.getSink(new Context(), "counter", "count");
    EventSource txtsrc = srcfact.getSource("asciisynth", "25", "100");
    counter.open();
    txtsrc.open();


    DirectDriver svrconn = new DirectDriver(rpcSrc, counter);
    svrconn.start();


    DirectDriver cliconn = new DirectDriver(txtsrc, rpcSink);
    cliconn.start();


    cliconn.join(Long.MAX_VALUE);
    Thread.sleep(250);


    svrconn.stop();
    rpcSink.close();
    rpcSrc.close();


    counter.close();
    txtsrc.close();


    LOG.info("read " + ((CounterSink) counter).getCount() + " lines");
    assertEquals(LINES, ((CounterSink) counter).getCount());
    assertNull(cliconn.getError());
    assertNull(svrconn.getError());
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of com.cloudera.flume.core.connector.DirectDriver$PumperThread

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.LogicalNode

com.cloudera.flume.conf.TestFactories

com.cloudera.flume.conf.TestFlumeBuilderFunctional

com.cloudera.flume.core.Event

com.cloudera.flume.core.EventSink

com.cloudera.flume.core.EventSource

com.cloudera.flume.handlers.debug.TestBenchmarkDeco

com.cloudera.flume.handlers.debug.TestChokeDecos

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.