Package com.cloudera.flume.core.connector

Examples of com.cloudera.flume.core.connector.DirectDriver$PumperThread


      }
    }

    // this will be replaceable with multi-threaded queueing versions or other
    // mechanisms
    driver = new DirectDriver("logicalNode " + nodeName, src, snk);
    this.state.state = NodeState.ACTIVE;
    driver.start();
    reconfigures.incrementAndGet();
  }
View Full Code Here


  @Test
  public void testNoneToNewFiles() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    drv.start();
    Clock.sleep(1000);

    genFiles(tmpdir, "foo", 10, 100);

    Clock.sleep(1000);

    assertEquals(1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);
  }
View Full Code Here

  public void testNoneToNewFilteredFiles() throws IOException,
      InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, "foo.*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    drv.start();
    Clock.sleep(1000);

    genFiles(tmpdir, "foo", 10, 100);
    genFiles(tmpdir, "bar", 15, 100);

    Clock.sleep(1000);

    assertEquals(1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);
  }
View Full Code Here

  @Test
  public void testExistingFiles() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    genFiles(tmpdir, "foo", 10, 100);

    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);
  }
View Full Code Here

    File tmpdir = FileUtil.mktempdir();
    File subDir = new File(tmpdir, "subdir");
    subDir.mkdirs();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    genFiles(tmpdir, "foo", 10, 100);

    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);

    // only did 10 files, ignored the dir.
    assertEquals(Long.valueOf(10),
        src.getMetrics().getLongMetric(TailDirSource.A_FILESADDED));
View Full Code Here

  @Test
  public void testExistingAddFiles() throws IOException, InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    genFiles(tmpdir, "foo", 10, 100);

    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());

    // more files show up
    genFiles(tmpdir, "foo", 10, 100);
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);

  }
View Full Code Here

  public void testExistingRemoveFiles() throws IOException,
      InterruptedException {
    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    genFiles(tmpdir, "foo", 10, 100);

    drv.start();
    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());

    FileUtil.rmr(tmpdir);

    Clock.sleep(1000);
    assertEquals(1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);
  }
View Full Code Here

    // generating files: emulating their existence prior to sink opening
    genFiles(tmpdir, "foo", 10, 100);

    TailDirSource src = new TailDirSource(tmpdir, ".*", true);
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    drv.start();
    Clock.sleep(1000);
    assertEquals(0, cnt.getCount());

    // adding lines to existing files
    addLinesToExistingFiles(tmpdir, 10);

    Clock.sleep(1000);
    assertEquals(10 * 10, cnt.getCount());

    // generating new files
    genFiles(tmpdir, "bar", 10, 100);

    Clock.sleep(1000);
    assertEquals(10 * 10 + 1000, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);

    // in total 20 files were added
    assertEquals(Long.valueOf(20),
        src.getMetrics().getLongMetric(TailDirSource.A_FILESADDED));
View Full Code Here

    subDirL3.mkdirs();
    genFiles(subDirL3, "2tail-old3", 10, 30);

    TailDirSource src = new TailDirSource(tmpdir, ".(2tail)?.*", true, 2);
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    drv.start();
    Clock.sleep(1000);
    assertEquals(0, cnt.getCount());

    // adding lines to existing files
    addLinesToExistingFiles(tmpdir, 10);
    addLinesToExistingFiles(subDirL1, 20);
    addLinesToExistingFiles(subDirL2, 30);
    addLinesToExistingFiles(subDirL3, 40);

    Clock.sleep(1000);
    int expEventsCount = 10 * 10 + 10 * 20 + 10 * 30;
    assertEquals(expEventsCount, cnt.getCount());

    // generating new files
    genFiles(tmpdir, "2tail-new", 10, 40);
    genFiles(subDirL1, "2tail-new1", 10, 30);
    genFiles(subDirL2, "2tail-new2", 10, 20);
    genFiles(subDirL3, "2tail-new3", 10, 10);

    Clock.sleep(1000);
    expEventsCount += 10 * 40 + 10 * 30 + 10 * 20;
    assertEquals(expEventsCount, cnt.getCount());

    // creating new subdir on level 2
    File newSubDirL2 = new File(subDirL1, "subdir-level2-new");
    newSubDirL2.mkdirs();
    genFiles(newSubDirL2, "2tail-new2-new", 10, 100);

    Clock.sleep(1000);
    expEventsCount += 10 * 100;
    assertEquals(expEventsCount, cnt.getCount());

    // deleting some dirs
    FileUtil.rmr(subDirL3);
    FileUtil.rmr(subDirL2);

    Clock.sleep(1000);
    assertEquals(expEventsCount, cnt.getCount());

    // adding back deleted dir, checking that it is caught up
    subDirL2.mkdirs();
    genFiles(subDirL2, "2tail-new2", 10, 100);
    Clock.sleep(1000);
    expEventsCount += 10 * 100;
    assertEquals(expEventsCount, cnt.getCount());

    drv.stop();
    FileUtil.rmr(tmpdir);

    ReportEvent report = src.getMetrics();
    assertEquals(Long.valueOf(80),
        report.getLongMetric(TailDirSource.A_FILESADDED));
View Full Code Here

    Assume.assumeTrue(!OSUtils.isWindowsOS());

    File tmpdir = FileUtil.mktempdir();
    TailDirSource src = new TailDirSource(tmpdir, ".*");
    AccumulatorSink cnt = new AccumulatorSink("tailcount");
    DirectDriver drv = new DirectDriver(src, cnt);

    drv.start();

    // This blows up when there are ~2000 files
    genFiles(tmpdir, "foo", 200, 10);
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());

    ReportEvent rpt1 = src.getMetrics();
    assertEquals(Long.valueOf(200),
        rpt1.getLongMetric(TailDirSource.A_FILESPRESENT));

    FileUtil.rmr(tmpdir); // This fails in windows because taildir keeps file
    // open
    tmpdir.mkdirs();
    Clock.sleep(1000);
    assertEquals(2000, cnt.getCount());

    ReportEvent rpt = src.getMetrics();
    assertEquals(rpt.getLongMetric(TailDirSource.A_FILESADDED),
        rpt.getLongMetric(TailDirSource.A_FILESDELETED));
    assertEquals(Long.valueOf(0),
        rpt.getLongMetric(TailDirSource.A_FILESPRESENT));

    drv.stop();
    FileUtil.rmr(tmpdir);
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.core.connector.DirectDriver$PumperThread

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.