Examples of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.

  public void testZKBackedConfigStore() throws IOException,
      InterruptedException {
    for (int i = 0; i < 10; ++i) {
      LOG.info("Opening ZK, attempt " + i);
      File tmp = FileUtil.mktempdir();
      FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
      cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
      cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
      ZooKeeperService.getAndInit(cfg);


      ZooKeeperConfigStore store = new ZooKeeperConfigStore();
      store.init();
      ConfigManager manager = new ConfigManager(store);
      manager.setConfig("foo", "my-test-flow", "null", "console");
      FlumeConfigData data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");
      store.shutdown();


      store = new ZooKeeperConfigStore();
      store.init();
      manager = new ConfigManager(store);
      data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");


      Map<String, FlumeConfigData> cfgs = new HashMap<String, FlumeConfigData>();
      String defaultFlowName = cfg.getDefaultFlowName();
      cfgs.put("bulk1", new FlumeConfigData(0, "s1", "sk1",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,
          "my-test-flow"));
      cfgs.put("bulk2", new FlumeConfigData(0, "s2", "sk2",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,

View Full Code Here

   */
  @Test
  public void testZKBackedConfigStoreNodes() throws IOException,
      InterruptedException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();

View Full Code Here

  /**
   * Test that watches are fired correctly for logical nodes
   */
  @Test
  public void testZBCSLogicalWatches() throws IOException, InterruptedException {
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.setBoolean(FlumeConfiguration.MASTER_ZK_USE_EXTERNAL, false);
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    ZooKeeperConfigStore store2 = new ZooKeeperConfigStore();

View Full Code Here

  /**
   * Test that watches are fired correctly for ChokeMap.
   */
  @Test
  public void testZBCSChokeWatches() throws IOException, InterruptedException {
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.setBoolean(FlumeConfiguration.MASTER_ZK_USE_EXTERNAL, false);
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    ZooKeeperConfigStore store2 = new ZooKeeperConfigStore();

View Full Code Here

   * Test disconnection
   */
  @Test
  public void testZBCSLoseZKCnxn() throws IOException, InterruptedException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    ZooKeeperService.getAndInit(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();
    store.setConfig("foo", "my-test-flow", "fab", "fat");


    ZooKeeperService.get().shutdown();
    Thread.sleep(30 * 1000);
    IOException ex = null;


    try {
      store.setConfig("foo", cfg.getDefaultFlowName(), "bar", "baz");
    } catch (IOException e) {
      ex = e;
    }
    assertNotNull("Expected IOException not thrown - still connected to ZK?",
        ex);

View Full Code Here

   */
  @Test
  public void testUnmapAllNodes() throws IOException, InterruptedException {


    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);
    ZooKeeperService.getAndInit(cfg);
    ZooKeeperConfigStore store = new ZooKeeperConfigStore();
    store.init();


    ConfigManager manager = new ConfigManager(store);

View Full Code Here

   */
  @Test
  public void testVersionIncrement() throws IOException, InterruptedException,
      KeeperException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);


    ZooKeeperService zk = new ZooKeeperService();
    zk.init(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore(zk);
    store.init();
    String defaultFlowName = cfg.getDefaultFlowName();
    store.setConfig("foo", defaultFlowName, "null", "baz");
    store.setConfig("foo2", defaultFlowName, "null", "baz");
    store.setConfig("foo3", defaultFlowName, "null", "baz");


    ZKClient client = zk.createClient();

View Full Code Here

  @Test
  @Ignore("Timing issue prevents this succeeding on Hudson")
  public void testLostSessionOK() throws IOException, InterruptedException,
      KeeperException {
    File tmp = FileUtil.mktempdir();
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.setInt(FlumeConfiguration.MASTER_SERVER_ID, 0);


    ZooKeeperService zk = new ZooKeeperService();
    zk.init(cfg);


    ZooKeeperConfigStore store = new ZooKeeperConfigStore(zk);
    store.init();
    String defaultFlowName = cfg.getDefaultFlowName();
    store.setConfig("foo", defaultFlowName, "bar", "baz");
    long sessionid = store.client.zk.getSessionId();
    byte[] sessionpass = store.client.zk.getSessionPasswd();


    // Force session expiration

View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testBZip2Codec() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "BZip2Codec");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

   * @throws InterruptedException
   */
  @Test
  public void testGzipOutputFormat() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.FaultyFlumeLocalAgentOOME

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestAgentSink

com.cloudera.flume.agent.TestAvroMultiMasterRPC

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestRPCMechanisms

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.