Examples of com.cloudera.flume.handlers.thrift.ThriftEventSource

Package com.cloudera.flume.handlers.thrift

Examples of com.cloudera.flume.handlers.thrift.ThriftEventSource

com.cloudera.flume.handlers.thrift.ThriftEventSource
This sets up the port that listens for incoming flume event rpc calls using Thrift.

         * I am hoping here that master calls this builder and at that time the
         * FlumeConfiguration is all set.
         */
        if (FlumeConfiguration.get().getEventRPC().equals(
            FlumeConfiguration.RPC_TYPE_THRIFT)) {
          return new ThriftEventSource(port);
        }
        if (FlumeConfiguration.get().getEventRPC().equals(
            FlumeConfiguration.RPC_TYPE_AVRO)) {
          return new AvroEventSource(port);
        }
        LOG.warn("event.rpc.type not defined.  It should be either "
            + "\"THRIFT\" or \"AVRO\". Defaulting to Thrift");
        return new ThriftEventSource(port);
      }
    };
  }

View Full Code Here


  final EventSource src;
  int port;


  public CollectorSource(int port) {
    this.src = new ThriftEventSource(port);
    this.port = port;
  }

View Full Code Here

    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());


    Thread t = new Thread() {
      public void run() {
        try {
          snk.open();
        } catch (IOException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");


    Thread.sleep(1000);
    tes.close();
    snk.close();
    t.interrupt();
    drain.interrupt();
    b.done();
  }

View Full Code Here

    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0", conf
        .getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

  @Test
  public void testConfirmBEChain() throws FlumeSpecException, IOException,
      InterruptedException {
    // create sources
    String c1 = "rpcSource(1234)";
    ThriftEventSource c1Src = (ThriftEventSource) FlumeBuilder.buildSource(
        LogicalNodeContext.testingContext(), c1);
    c1Src.open();


    String c2 = "rpcSource(1235)";
    ThriftEventSource c2Src = (ThriftEventSource) FlumeBuilder.buildSource(
        LogicalNodeContext.testingContext(), c2);
    c2Src.open();


    // create agentBEChain sink
    String spec = "agentBEChain(\"localhost:1234\", \"localhost:1235\")";
    EventSink snk = new CompositeSink(new Context(), spec);
    snk.open();


    Event e1 = new EventImpl("test 1".getBytes());
    Event e2 = new EventImpl("test 2".getBytes());
    Event e3 = new EventImpl("test 3".getBytes());
    Event e4 = new EventImpl("test 4".getBytes());


    // Everything is on and we send some messages
    snk.append(e1);
    Clock.sleep(100);
    LOG.info(c1Src.getMetrics().toString());
    assertEquals(1,
        (long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    c1Src.next();
    c1Src.close();


    // Killed the first of the chain, should go to backup
    // the number of events lost here is not consistent after close. this
    // seems time based, and the first two seem to be lost
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    snk.append(e3);
    Clock.sleep(20);
    snk.append(e4);
    Clock.sleep(20);


    LOG.info(c2Src.getMetrics().toString());
    assertEquals(2,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // 2 lost in network buffer, but two received in backup.  yay.
    c2Src.next();
    c2Src.next();
    c2Src.close();


    // all thrift sinks are closed now, we should loss messages
    snk.append(e1); // lost
    Clock.sleep(20);
    snk.append(e2); // lost
    Clock.sleep(20);
    snk.append(e3); // lost
    Clock.sleep(20);
    snk.append(e4); // lost
    Clock.sleep(20);


    // re-open desination 1.
    c1Src.open();
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    c1Src.next();
    c1Src.close();
    LOG.info(c1Src.getMetrics().toString());
    // 2 events from prevoius + 1 from new open
    // first one fails on reopen but next succeeds
    assertEquals(2 + 1,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    snk.close();
  }

View Full Code Here

  @Test
  public void testConfirmDFOChain() throws FlumeSpecException, IOException,
      InterruptedException {
    // create sources
    String c1 = "rpcSource(1234)";
    ThriftEventSource c1Src = (ThriftEventSource) FlumeBuilder.buildSource(
        LogicalNodeContext.testingContext()


        , c1);
    c1Src.open();


    String c2 = "rpcSource(1235)";
    ThriftEventSource c2Src = (ThriftEventSource) FlumeBuilder.buildSource(
        LogicalNodeContext.testingContext(), c2);
    c2Src.open();


    // create agentDFOChain sink
    File tmpDir = FileUtil.mktempdir();
    String spec = "agentDFOChain(\"localhost:1234\", \"localhost:1235\")";
    CompositeSink snk = new CompositeSink(new LogicalNodeContext(
        tmpDir.getName(), tmpDir.getName()), spec);
    snk.open();


    Event e1 = new EventImpl("test 1".getBytes());
    Event e2 = new EventImpl("test 2".getBytes());
    Event e3 = new EventImpl("test 3".getBytes());
    Event e4 = new EventImpl("test 4".getBytes());


    // Everything is on and we send some messages.
    snk.append(e1);
    Clock.sleep(100);
    LOG.info(c1Src.getMetrics().toString());
    assertEquals(1,
        (long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // it got through, yay.
    c1Src.next();
    c1Src.close();


    // Killed the first of the chain, should go to backup
    // the number of events lost here is not consistent after close. this
    // seems time based, and the first two seem to be lost
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    snk.append(e3);
    Clock.sleep(20);
    snk.append(e4);
    Clock.sleep(20);
       
    LOG.info(c2Src.getMetrics().toString());
    assertEquals(2,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // 2 lost in network buffer, but two received in backup. yay.
    c2Src.next();
    c2Src.next();
    c2Src.close();


    // all thrift sinks are closed now, we should end up in dfo
    snk.append(e1); // lost in thrift sink buffer
    Clock.sleep(20);
    snk.append(e2); // lost in thrift sink buffer
    Clock.sleep(20);
    snk.append(e3); // written
    Clock.sleep(20);
    snk.append(e4); // written
    Clock.sleep(20);
    LOG.info(snk.getMetrics().toText());
    assertEquals(
        2,
        (long) ReportUtil.getFlattenedReport(snk).getLongMetric(
            "backup.DiskFailover.NaiveDiskFailover.writingEvts"));


    // re-open destination 1
    c1Src.open();
    snk.append(e1);
    Clock.sleep(20);
    snk.append(e2);
    Clock.sleep(20);
    c1Src.next();


    // get handle to roller in dfo log roller to provide data
    AgentFailChainSink afcs = (AgentFailChainSink) snk.getSink();
    BackOffFailOverSink bofos = (BackOffFailOverSink) ((CompositeSink) afcs.snk)
        .getSink();
    DiskFailoverDeco dfo = (DiskFailoverDeco) bofos.getBackup();
    DiskFailoverManager dfm = dfo.getFailoverManager();
    RollSink dfoWriter = dfo.getDFOWriter();
    dfoWriter.rotate(); // allow dfo retry thread to go.


    // give data some time to show up.
    Clock.sleep(1000);
    c1Src.next();
    c1Src.next();
    c1Src.next();
    c1Src.next();
    c1Src.close();
    ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
    LOG.info(rpt.toString());


    String written = "backup.DiskFailover.NaiveDiskFailover.writingEvts";
    assertEquals(4, (long) rpt.getLongMetric(written));
    // yay. all four events written to dfo log


    String primary = "backup.DiskFailover."
        + "LazyOpenDecorator.InsistentAppend.StubbornAppend."
        + "InsistentOpen.FailoverChainSink.sentPrimary";
    assertEquals(4, (long) rpt.getLongMetric(primary));
    // yay all four go through to the path we wanted. (the primary after the
    // disk failover)


    // data from DFO log was sent.
    assertEquals(2 + 4,
        (long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
    // first one fails on reopen but next succeeds
    assertEquals(2 + 1,
        (long) c2Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));


  }

View Full Code Here

    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink snk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) snk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));
    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 10, 10000000);
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) tsnk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

    b.mark("begin");
    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
    // need to drain the sink otherwise its queue will fill up with events!
    Thread drain = new Thread("drain") {
      public void run() {
        try {
          EventUtil.dumpAll(tes, new NullSink());
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    drain.start(); // drain the sink.
    b.mark("receiver_started");


    final ThriftEventSink tsnk = new ThriftEventSink("0.0.0.0",
        conf.getCollectorPort());
    // make size happen first all the time.
    final BatchingDecorator snk = new BatchingDecorator(tsnk, 100, 10000000);
    snk.open();
    b.mark("sink_started");


    EventUtil.dumpAll(mem, snk);
    b.mark("thrift sink to thrift source done");
    // MB/s = B/us
    b.mark("MB/s", (double) tsnk.getSentBytes()
        / (double) (b.getLastDelta() / 1000));


    tes.close();
    snk.close();
    drain.interrupt();
    b.done();
  }

View Full Code Here

0 1

TOP

Related Classes of com.cloudera.flume.handlers.thrift.ThriftEventSource

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSource

com.cloudera.flume.conf.SourceFactory.SourceBuilder

com.cloudera.flume.core.Event

com.cloudera.flume.handlers.rpc.RpcSource

com.cloudera.flume.PerfThriftSinks

com.cloudera.flume.reporter.ReportEvent

org.apache.thrift.protocol.TBinaryProtocol.Factory

org.apache.thrift.server.TSaneThreadPoolServer

org.apache.thrift.transport.TSaneServerSocket

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.