Package com.cloudera.flume.master

Examples of com.cloudera.flume.master.ConfigurationManager


    buildMaster();

    // Instead of loading from a ZK Store, we just see the config in the "deep"
    // config manager. Any translations will not occur.
    ConfigurationManager loaded = cfgMan;
    loaded.setConfig("coll11", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll12", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll13", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll14", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("agent1", "flow1", "null", "autoBEChain");

    loaded.setConfig("coll21", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll22", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll23", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll24", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("agent2", "flow2", "null", "autoBEChain");

    // this is the outer configman, should have no translation.
    ConfigurationManager cfgman1 = flumeMaster.getSpecMan();
    Map<String, FlumeConfigData> cfgs1 = cfgman1.getTranslatedConfigs();
    assertEquals(0, cfgs1.size()); // no translations happened

    // start the master (which should trigger an update and translation
    flumeMaster.serve();
  }
View Full Code Here


    buildMaster();

    // Instead of loading from a ZK Store, we just see the config in the "deep"
    // config manager. Any translations will not occur.
    ConfigurationManager loaded = cfgMan;
    loaded.setConfig("coll11", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll12", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll13", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("coll14", "flow1", "autoCollectorSource", "null");
    loaded.setConfig("agent1", "flow1", "null", "autoBEChain");

    loaded.setConfig("coll21", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll22", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll23", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("coll24", "flow2", "autoCollectorSource", "null");
    loaded.setConfig("agent2", "flow2", "null", "autoBEChain");

    // this is the outer configman, should have no translation.
    ConfigurationManager cfgman1 = flumeMaster.getSpecMan();
    Map<String, FlumeConfigData> cfgs1 = cfgman1.getTranslatedConfigs();
    assertEquals(0, cfgs1.size()); // no translations happened

    // start the master (which should trigger an update and translation
    flumeMaster.serve();
View Full Code Here

  /**
   * Instantiate and expose managers
   */
  void setupNewManagers() {
    ConfigurationManager parent = new ConfigManager();
    ConfigurationManager self = new ConfigManager();
    FailoverChainManager fcMan = new ConsistentHashFailoverChainManager(3);
    ConfigurationManager self2 = new ConfigManager();

    failover = new FailoverConfigurationManager(parent, self2, fcMan);
    statman = new StatusManager();
    logical = new LogicalConfigurationManager(failover, self, statman);
    trans = logical;
View Full Code Here

  /**
   * Test the core of the LogicalConfigManager
   */
  @Test
  public void testLogicalTrans() throws IOException, FlumeSpecException {
    ConfigurationManager parent = new ConfigManager();
    ConfigurationManager self = new ConfigManager();
    StatusManager statman = new StatusManager();
    ConfigurationManager trans = new LogicalConfigurationManager(parent, self,
        statman);

    // make it so that the local host info is present
    statman.updateHeartbeatStatus("foo", "foo", "foo", NodeState.HELLO, Clock
        .unixTime());
    statman.updateHeartbeatStatus("bar", "bar", "bar", NodeState.HELLO, Clock
        .unixTime());

    // Next spawn so that all are mapped onto a node and now gets a physical
    trans.addLogicalNode("foo", "foo");
    trans.addLogicalNode("bar", "bar");

    // now set configs
    trans.setConfig("foo", DEFAULTFLOW, "logicalSource", "null");
    trans.setConfig("bar", DEFAULTFLOW, "null", "logicalSink(\"foo\")");

    // update the configurations
    trans.updateAll();
    int port = FlumeConfiguration.get().getCollectorPort();

    // check if translated
    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals("rpcSink( \"foo\", " + port + " )", transData.getSinkConfig());

    FlumeConfigData transData2 = trans.getConfig("foo");
    assertEquals("rpcSource( " + port + " )", transData2.getSourceConfig());
    assertEquals("null", transData2.getSinkConfig());

    // self is same as translated
    FlumeConfigData selfData = self.getConfig("bar");
View Full Code Here

   * Test the core of the LogicalConfigManager when it fails
   */
  @Test
  public void testLogicalTransFailSource() throws IOException,
      FlumeSpecException {
    ConfigurationManager parent = new ConfigManager();
    ConfigurationManager self = new ConfigManager();
    StatusManager statman = new StatusManager();
    ConfigurationManager trans = new LogicalConfigurationManager(parent, self,
        statman);

    // now set configs
    trans.setConfig("foo", DEFAULTFLOW, "logicalSource", "null");
    trans.setConfig("bar", DEFAULTFLOW, "null", "logicalSink(\"foo\")");

    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertTrue(transData.getSinkConfig().contains("fail"));

    FlumeConfigData transData2 = trans.getConfig("foo");
    assertTrue(transData2.getSourceConfig().contains("fail"));
    assertEquals("null", transData2.getSinkConfig());

  }
View Full Code Here

   * Test to ensure attempts to map a single logical node to multiple physicals
   * is discarded with a warning. This may be exception worthy.
   */
  @Test
  public void testDuplicateAssignment() {
    ConfigurationManager parent = new ConfigManager();
    ConfigurationManager self = new ConfigManager();
    StatusManager statman = new StatusManager();
    ConfigurationManager trans = new LogicalConfigurationManager(parent, self,
        statman);

    trans.addLogicalNode("hostA", "foo");
    assertEquals(1, trans.getLogicalNodeMap().size());

    trans.addLogicalNode("hostB", "foo");
    assertEquals(1, trans.getLogicalNodeMap().size());

    assertEquals(Arrays.asList("foo"), trans.getLogicalNode("hostA"));
    assertEquals(Collections.EMPTY_LIST, trans.getLogicalNode("hostB"));
  }
View Full Code Here

  /**
   * Returns a config man for a flow (never returns null). This is not thread
   * safe and must be run guarded by the this lock.
   */
  ConfigurationManager getCreateFlowConfigMan(String flowid) {
    ConfigurationManager cfg = flows.get(flowid);
    if (cfg != null) {
      return cfg;
    }

    cfg = createConfigMan();
View Full Code Here

  synchronized public boolean addLogicalNode(String physNode, String logicNode) {
    boolean result;

    result = parent.addLogicalNode(physNode, logicNode);
    String flowid = getFlowId(logicNode);
    ConfigurationManager fcfg = getCreateFlowConfigMan(flowid);
    fcfg.addLogicalNode(physNode, logicNode);

    return result;
  }
View Full Code Here

    }

    // do a bulk update for each sub flow config manager
    for (Entry<String, Map<String, FlumeConfigData>> e : flowsets.entrySet()) {
      String flow = e.getKey();
      ConfigurationManager fcfg = getCreateFlowConfigMan(flow);
      fcfg.setBulkConfig(e.getValue());
    }
  }
View Full Code Here

  }

  @Override
  synchronized public FlumeConfigData getConfig(String logicNode) {
    String flowid = getFlowId(logicNode);
    ConfigurationManager fcfg = getCreateFlowConfigMan(flowid);
    return fcfg.getConfig(logicNode);
  }
View Full Code Here

TOP

Related Classes of com.cloudera.flume.master.ConfigurationManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.