Package com.cloudera.flume.handlers.debug

Examples of com.cloudera.flume.handlers.debug.MemorySinkSource


  public void testThriftSendMulti() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
View Full Code Here


  public void testThriftRawSend() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    final ThriftEventSource tes = new ThriftEventSource(conf.getCollectorPort());
    tes.open();
View Full Code Here

  public void testCopy() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile copy");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    File tmp = File.createTempFile("test", "tmp");
    tmp.deleteOnExit();
    SeqfileEventSink sink = new SeqfileEventSink(tmp);
View Full Code Here

  public void testDirectWrite() throws IOException, InterruptedException {

    Benchmark b = new Benchmark("hdfs seqfile write");
    b.mark("begin");

    MemorySinkSource mem = FlumeBenchmarkHarness.synthInMem();
    b.mark("disk_loaded");

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);

    Writer w = SequenceFile.createWriter(hdfs, conf, path,
        WriteableEventKey.class, WriteableEvent.class);
    b.mark("hdfs_fileopen_started");

    Event e = null;
    while ((e = mem.next()) != null) {
      // writing
      w.append(new WriteableEventKey(e), new WriteableEvent(e));
    }
    w.close();
    b.mark("seqfile_hdfs_write");
View Full Code Here

  public static MemorySinkSource synthInMem(int count, int bodySz, int seed)
      throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(count, bodySz, seed);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    return mem;
  }
View Full Code Here

   * @throws InterruptedException
   */
  MemorySinkSource setupAckRoll() throws IOException, InterruptedException {

    // we can roll now.
    MemorySinkSource ackedmem = new MemorySinkSource();
    // simulate the stream coming from an e2e acking agent.
    AckChecksumInjector<EventSink> inj = new AckChecksumInjector<EventSink>(
        ackedmem);
    inj.open();
    inj.append(new EventImpl("foo 1".getBytes()));
View Full Code Here

      final int COUNT = 100;
      final int ROLLS = 1000;

      // setup a source of data that will shove a lot of ack laden data into the
      // stream.
      MemorySinkSource mem = new MemorySinkSource();
      EventSource src = new NoNlASCIISynthSource(COUNT, 10);
      src.open();
      Event e;
      while ((e = src.next()) != null) {
        AckChecksumInjector<EventSink> acks = new AckChecksumInjector<EventSink>(
View Full Code Here

      FlumeSpecException, InterruptedException {
    File f = FileUtil.createTempFile("multitemp1", ".tmp");
    f.deleteOnExit();
    File f2 = FileUtil.createTempFile("multitemp2", ".tmp");
    f2.deleteOnExit();
    final MemorySinkSource snk = new MemorySinkSource();
    final EventSource src = TailSource.multiTailBuilder().build(
        LogicalNodeContext.testingContext(), f.getAbsolutePath(),
        f2.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 60;
    Thread t = new Thread() {
      @Override
      public void run() {
        try {
          src.open();
          snk.open();
          EventUtil.dumpN(count, src, snk);
          src.close();
          snk.close();
          done.countDown();
        } catch (IOException e) {
          e.printStackTrace();
        } catch (InterruptedException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
    };
    t.start();

    int log1 = 0, log2 = 0;
    FileWriter fw = new FileWriter(f);
    FileWriter fw2 = new FileWriter(f2);
    for (int i = 0; i < count; i++) {
      if (Math.random() > 0.5) {
        fw.append("Line " + i + "\n");
        fw.flush();
        log1++;
      } else {
        fw2.append("Line " + i + "\n");
        fw2.flush();
        log2++;
      }
    }
    fw.close();
    fw2.close();

    assertTrue(done.await(15, TimeUnit.SECONDS));

    Event e = null;
    while ((e = snk.next()) != null) {
      byte[] fn = e.get(TailSource.A_TAILSRCFILE);
      String sfn = new String(fn);
      if (!sfn.equals(f.getName()) && !sfn.equals(f2.getName())) {
        Assert.fail("didn't have tail src file metadata! " + sfn + " != "
            + f.getName() + " or " + f2.getName());
View Full Code Here

    Attributes.setString(e, "duped", "second");
  }

  @Test
  public void testMaskNoConflict() throws IOException, InterruptedException {
    MemorySinkSource mem = new MemorySinkSource();
    EventSink s1 = new ValueDecorator<EventSink>(mem, "duped", "second");
    EventSink s2 = new MaskDecorator<EventSink>(s1, "duped");
    EventSink snk = new ValueDecorator<EventSink>(s2, "duped", "first");
    snk.open();

    Event e = new EventImpl("foo".getBytes());
    snk.append(e);
    snk.close();

    Event e2 = mem.next();
    assertEquals("second", Attributes.readString(e2, "duped"));
  }
View Full Code Here

   * this isn't possible via the configuration language currently.
   */
  public void sequenceFileWriteReadHelper(String... args) throws IOException,
      FlumeSpecException, InterruptedException {

    MemorySinkSource mem = MemorySinkSource.cannedData("test ", 5);

    // setup sink.
    File f = FileUtil.createTempFile("sequencefile", ".seq");
    f.deleteOnExit();
    FileOutputStream fos = new FileOutputStream(f);
    LOG.info("filename before escaping: " + f.getAbsolutePath());
    OutputFormat out = FormatFactory.get().getOutputFormat("seqfile", args);
    mem.open();
    Event e = mem.next();
    while (e != null) {
      out.format(fos, e);
      e = mem.next();
    }

    mem.open();
   
    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem fs = FileSystem.getLocal(conf);
    SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(f.toURI()), conf);
    WriteableEventKey k = new WriteableEventKey();
    WriteableEvent evt = new WriteableEvent();
    while (r.next(k, evt)) {
      Event expected = mem.next();
      assertEquals(evt.getTimestamp(), expected.getTimestamp());
      assertEquals(evt.getNanos(), expected.getNanos());
      assertEquals(evt.getPriority(), expected.getPriority());
      assertTrue(Arrays.equals(evt.getBody(), expected.getBody()));
    }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.handlers.debug.MemorySinkSource

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.