Examples of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.

    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);

View Full Code Here

    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());


    Thread t = new Thread() {
      public void run() {
        try {

View Full Code Here

    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);

View Full Code Here

  public void test3NodesDone() throws InterruptedException,
      TTransportException, IOException {
    assertEquals(0, flumeMaster.getSpecMan().getAllConfigs().size());


    String nodename = "foo";
    FlumeConfiguration conf = FlumeConfiguration.createTestableConfiguration();
    FlumeNode n = new FlumeNode(conf, nodename,
        new DirectMasterRPC(flumeMaster), false, false);
    n.start();


    String node2 = "bar";

View Full Code Here

   * Tries to connect to several servers in turn and compensate as masters fail.
   */
  @Test
  public void testConnect() throws TTransportException, IOException,
      InterruptedException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.MASTER_HEARTBEAT_SERVERS,
        "localhost:9999,localhost:56789,localhost:56790");
    conf.set(FlumeConfiguration.MASTER_HEARBEAT_RPC, "AVRO");
    MultiMasterRPC masterRPC = new MultiMasterRPC(conf, false);
    MockAvroServer server1 = new MockAvroServer();
    server1.serve(56789);
    MockAvroServer server2 = new MockAvroServer();
    server2.serve(56790);

View Full Code Here

    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();


    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.


    // mem -> AvroEventSink
    AvroEventSink snk = new AvroEventSink("0.0.0.0",
        conf.getCollectorPort() + 1);
    snk.open();
    EventUtil.dumpAll(mem, snk);
    mem.close();
    snk.close();

View Full Code Here

   */
  @Test
  public void testManyThreadsAvroSend() throws IOException,
      InterruptedException {
    final int threads = 10;
    final FlumeConfiguration conf = FlumeConfiguration.get();
    // this is a slight tweak to avoid port conflicts
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();


    final CounterSink cnt = new CounterSink("count");
    cnt.open();
    Thread t = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, cnt);
        } catch (Exception e) {
        }
      }
    };
    t.start(); // drain the sink.


    // fork off threads threads and have them start all the same time.
    final CountDownLatch sendStarted = new CountDownLatch(threads);
    final CountDownLatch sendDone = new CountDownLatch(threads);
    final AtomicLong sendByteSum = new AtomicLong(0);
    for (int i = 0; i < threads; i++) {
      final int id = i;
      Thread th = new Thread() {
        public void run() {
          try {
            EventSource txt = new NoNlASCIISynthSource(25, 100);
            txt.open();
            MemorySinkSource mem = new MemorySinkSource();
            mem.open();
            EventUtil.dumpAll(txt, mem);
            txt.close();


            // mem -> AvroEventSink
            AvroEventSink snk = new AvroEventSink("0.0.0.0", conf
                .getCollectorPort() + 1);
            snk.open();


            sendStarted.countDown();
            sendStarted.await();

View Full Code Here


  /**
   * This also implements the Apache Commons Daemon interface's start
   */
  synchronized public void start() {
    FlumeConfiguration conf = FlumeConfiguration.get();


    // the simple report interface
    simpleReportManager.add(vmInfo);
    simpleReportManager.add(sysInfo);
    simpleReportManager.add(new Reportable() {


      @Override
      public String getName() {
        return FlumeNode.this.getName();
      }


      @Override
      public ReportEvent getMetrics() {
        return FlumeNode.this.getReport();
      }


      @Override
      public Map<String, Reportable> getSubMetrics() {
        return ReportUtil.noChildren();
      }
    });


    // the full report interface
    ReportManager.get().add(vmInfo);
    ReportManager.get().add(sysInfo);
    ReportManager.get().add(this);


    if (startHttp) {
      try {
        String webPath = getWebPath(conf);


        boolean findport = FlumeConfiguration.get().getNodeAutofindHttpPort();
        this.http = new StatusHttpServer("flumeagent", webPath, "0.0.0.0", conf
            .getNodeStatusPort(), findport);
        http.addServlet(jerseyNodeServlet(), "/node/*");
        http.start();
      } catch (IOException e) {
        LOG.error("Flume node failed: " + e.getMessage(), e);

View Full Code Here

  /**
   * Checks to verify that a Avro server doesn't hang forever on closing
   */
  @Test
  public void testAvroEventServerCloseTimeout() throws IOException {
    final FlumeConfiguration conf = FlumeConfiguration.get();
    // this is a slight tweak to avoid port conflicts
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();
    tes.enqueue(new EventImpl(new byte[0]));
    tes.close();
  }

View Full Code Here

   */
  public static void nodeConfigChecksOk() throws IOException {
    // TODO (jon) if we add more checks in here, make the different managers
    // responsible for throwing an Exception on construction instead.


    FlumeConfiguration conf = FlumeConfiguration.get();


    String s = conf.getAgentLogsDir();
    File f = new File(s);


    if (!FileUtil.makeDirs(f)) {
      throw new IOException("Path to Log dir cannot be created: '" + s
          + "'.  Check permissions?");

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.FaultyFlumeLocalAgentOOME

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestAgentSink

com.cloudera.flume.agent.TestAvroMultiMasterRPC

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestRPCMechanisms

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.