Examples of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.

  public void testOpenClose() throws Exception {
    // Set directory of webapps to build-specific dir
    FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
        "build/webapps");


    FlumeConfiguration conf = FlumeConfiguration.get();
    String webPath = FlumeNode.getWebPath(conf);
    int port = FlumeConfiguration.get().getNodeStatusPort();
    StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
        "0.0.0.0", port, false);

View Full Code Here

  public void testAutoFindPort() throws IOException, Exception {
    // Set directory of webapps to build-specific dir
    FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
        "build/webapps");


    FlumeConfiguration conf = FlumeConfiguration.get();
    String webPath = FlumeNode.getWebPath(conf);
    int port = FlumeConfiguration.get().getNodeStatusPort();
    StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
        "0.0.0.0", port, true);
    http.start();

View Full Code Here

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      Assert.fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.clear(); // reset all back to defaults.
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    mock = new MockMasterRPC();
    node = new FlumeNode(mock, false /* starthttp */, false /* oneshot */);
    ReportManager.get().clear();
  }

View Full Code Here

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      Assert.fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    mock = new MockMasterRPC();


    nodesMan = (nodesMan != null) ? nodesMan : new LogicalNodeManager(NetUtils
        .localhost());
    walMan = (walMan != null) ? walMan : new NaiveFileWALManager(new File(conf
        .getAgentLogsDir()));
    dfMan = (dfMan != null) ? dfMan : new NaiveFileFailoverManager(new File(
        conf.getAgentLogsDir()));
    colAck = (colAck != null) ? colAck : new CollectorAckListener(mock);
    liveman = (liveman != null) ? liveman : new LivenessManager(nodesMan, mock,
        walMan);


    node = new FlumeNode(NetUtils.localhost(), mock, nodesMan, walMan, dfMan,

View Full Code Here

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");

View Full Code Here

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 10, 10000000);
    snk.open();
    b.mark("sink_started");

View Full Code Here

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 100, 10000000);
    snk.open();
    b.mark("sink_started");

View Full Code Here

    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());


    Thread t = new Thread() {
      public void run() {
        try {
          snk.open();

View Full Code Here

    b.mark("begin");


    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");

View Full Code Here


    b.mark("local_disk_write done");


    sink.close();


    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
    hdfs.deleteOnExit(dst);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.FaultyFlumeLocalAgentOOME

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestAgentSink

com.cloudera.flume.agent.TestAvroMultiMasterRPC

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestRPCMechanisms

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.