Examples of com.cloudera.flume.conf.FlumeConfigData

com.cloudera.flume.conf.FlumeConfigData
Stores configuration data for a Flume agent. Since we use multiple RPC frameworks with slightly different data formats, this class provides translation to and from those transport-specific types.

    setupCollectorAgentConfigs();


    // update the configurations
    trans.updateAll();


    FlumeConfigData transData0 = trans.getConfig("bar");
    assertEquals("null", transData0.getSourceConfig());
    String lh = NetUtils.localhost();
    String first = "< { lazyOpen => fail( \"logicalSink( \\\"foo\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    assertEquals(first, transData0.getSinkConfig());


    // update one at a time and check config
    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo",
        NodeState.HELLO, Clock.unixTime());
    trans.updateAll(); // TODO remove
    String second = "< { lazyOpen => rpcSink( \""
        + lh
        + "\", 35853 ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(second, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // update one at a time and check config
    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo2",
        NodeState.HELLO, Clock.unixTime());
    trans.updateAll();
    String third = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";
    transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(third, transData.getSinkConfig());


    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35855 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo3",
        NodeState.HELLO, Clock.unixTime());
    trans.updateAll();


    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // intermediate data
    FlumeConfigData failData = failover.getConfig("bar");
    assertEquals("null", failData.getSourceConfig());
    String failTranslated = "< { lazyOpen => logicalSink( \"foo\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo3\" ) } ? "
        + "< { lazyOpen => logicalSink( \"foo2\" ) } ? null > > >";
    assertEquals(failTranslated, failData.getSinkConfig());
  }

View Full Code Here


      ZooKeeperConfigStore store = new ZooKeeperConfigStore();
      store.init();
      ConfigManager manager = new ConfigManager(store);
      manager.setConfig("foo", "my-test-flow", "null", "console");
      FlumeConfigData data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");
      store.shutdown();


      store = new ZooKeeperConfigStore();
      store.init();
      manager = new ConfigManager(store);
      data = manager.getConfig("foo");
      assertEquals(data.getSinkConfig(), "console");
      assertEquals(data.getSourceConfig(), "null");


      Map<String, FlumeConfigData> cfgs = new HashMap<String, FlumeConfigData>();
      String defaultFlowName = cfg.getDefaultFlowName();
      cfgs.put("bulk1", new FlumeConfigData(0, "s1", "sk1",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,
          "my-test-flow"));
      cfgs.put("bulk2", new FlumeConfigData(0, "s2", "sk2",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,
          defaultFlowName));
      store.bulkSetConfig(cfgs);


      data = manager.getConfig("bulk1");
      assertEquals(data.getSinkConfig(), "sk1");
      assertEquals(data.getSourceConfig(), "s1");
      assertEquals(data.getFlowID(), "my-test-flow");


      data = manager.getConfig("bulk2");
      assertEquals(data.getSinkConfig(), "sk2");
      assertEquals(data.getSourceConfig(), "s2");


      cfgs.put("bulk1", new FlumeConfigData(0, "s3", "sk3",
          LogicalNode.VERSION_INFIMUM, LogicalNode.VERSION_INFIMUM,
          defaultFlowName));
      cfgs.remove("bulk2");
      store.bulkSetConfig(cfgs);


      // Check that unchanged configs persist
      data = manager.getConfig("bulk2");
      assertEquals(data.getSinkConfig(), "sk2");
      assertEquals(data.getSourceConfig(), "s2");
      assertEquals(data.getFlowID(), defaultFlowName);
      store.shutdown();


      ZooKeeperService.get().shutdown();
      FileUtil.rmr(tmp);
    }

View Full Code Here

    setupCollectorAgentConfigs();


    // update the configurations
    trans.updateAll();


    FlumeConfigData transData0 = trans.getConfig("bar");
    assertEquals("null", transData0.getSourceConfig());
    String lh = NetUtils.localhost();
    String first = "< { lazyOpen => fail( \"logicalSink( \\\"foo\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    assertEquals(first, transData0.getSinkConfig());


    // update one at a time and check config
    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo",
        NodeState.HELLO, Clock.unixTime());
    trans.updateAll(); // TODO remove
    String second = "< { lazyOpen => rpcSink( \""
        + lh
        + "\", 35853 ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(second, transData.getSinkConfig());


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // We knock out a node. What is the right behavior? Use old information.
    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo",
        NodeState.LOST, Clock.unixTime());
    trans.updateAll(); // TODO remove
    String third = "< { lazyOpen => rpcSink( \""
        + lh
        + "\", 35853 ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(third, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());
  }

View Full Code Here

   * Test that Avro-based serialization of node configs works
   */
  @Test
  public void testSerializeConfigs() throws IOException {
    Map<String, FlumeConfigData> cfgmap = new HashMap<String, FlumeConfigData>();
    FlumeConfigData fcd = new FlumeConfigData();
    fcd.flowID = "my-flow";
    fcd.sinkConfig = "my-sink";
    fcd.sourceConfig = "my-source";
    fcd.timestamp = 10L;
    fcd.sinkVersion = 10;

View Full Code Here

    String lh = NetUtils.localhost();
    String translated = "< { lazyOpen => rpcSink( \"" + lh + "\", 35853 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35855 ) } ? "
        + "< { lazyOpen => rpcSink( \"" + lh + "\", 35854 ) } ? null > > >";


    FlumeConfigData transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    statman.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo3",
        NodeState.HELLO, Clock.unixTime());
    trans.updateAll();


    LOG.info("Full Translation: " + trans);
    LOG.info("logical Translation: " + logical);
    LOG.info("failover Translation: " + failover);


    FlumeConfigData transData = trans.getConfig("bar");
    assertEquals("null", transData.getSourceConfig());
    assertEquals(translated, transData.getSinkConfig());


    transCollData = trans.getConfig("foo");
    assertEquals("rpcSource( 35853 )", transCollData.getSourceConfig());
    assertEquals("null", transCollData.getSinkConfig());


    // //
    // Unmap all the nodes.
    // //
    trans.unmapAllLogicalNodes();


    // This should go back to the failed state
    FlumeConfigData transData0 = trans.getConfig("bar");
    assertEquals("null", transData0.getSourceConfig());
    String first = "< { lazyOpen => fail( \"logicalSink( \\\"foo\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo3\\\" )\" ) } ? "
        + "< { lazyOpen => fail( \"logicalSink( \\\"foo2\\\" )\" ) } ? null > > >";
    assertEquals(first, transData0.getSinkConfig());


  }

View Full Code Here

    tmp.delete();
    tmp.deleteOnExit();
    MemoryBackedConfigStore store = new MemoryBackedConfigStore();
    ConfigManager manager = new ConfigManager(store);
    manager.setConfig("foo", "my-test-flow", "null", "console");
    FlumeConfigData data = manager.getConfig("foo");
    assertEquals(data.getSinkConfig(), "console");
    assertEquals(data.getSourceConfig(), "null");


    manager.saveConfigFile(tmp.getAbsolutePath());


    manager = new ConfigManager(new MemoryBackedConfigStore());
    manager.loadConfigFile(tmp.getAbsolutePath());
    data = manager.getConfig("foo");
    assertEquals(data.getSinkConfig(), "console");
    assertEquals(data.getSourceConfig(), "null");
  }

View Full Code Here


  /**
   * returns the flow id of a particular logicalNode
   */
  synchronized public String getFlowId(String logicalNode) {
    FlumeConfigData fcd = parent.getConfig(logicalNode);
    if (fcd == null) {
      return null;
    }
    return fcd.flowID;
  }

View Full Code Here

    // create buckets for each flow id.
    // flow -> (ln, fcd)
    Map<String, Map<String, FlumeConfigData>> flowsets = new HashMap<String, Map<String, FlumeConfigData>>();
    for (Entry<String, FlumeConfigData> e : configs.entrySet()) {
      String node = e.getKey();
      FlumeConfigData fcd = e.getValue();
      String flow = fcd.flowID;
      Map<String, FlumeConfigData> cfgs = flowsets.get(flow);
      if (cfgs == null) {
        cfgs = new HashMap<String, FlumeConfigData>();
        flowsets.put(flow, cfgs);

View Full Code Here

  synchronized public void loadConfigFile(String from) throws IOException {
    ConfigManager cfgs = new ConfigManager();
    cfgs.loadConfigFile(from);
    String defaultFlow = FlumeConfiguration.get().getDefaultFlowName();
    for (Entry<String, FlumeConfigData> e : cfgs.getAllConfigs().entrySet()) {
      FlumeConfigData fcd = e.getValue();
      // Flow names aren't saved with this mechanism.
      try {
        setConfig(e.getKey(), defaultFlow, fcd.sourceConfig, fcd.sinkConfig);
      } catch (FlumeSpecException e1) {
        throw new IOException(e1.getMessage(), e1);

View Full Code Here

  @Override
  synchronized public void refresh(String logicalNode) throws IOException {
    String oldflow = getFlowId(logicalNode);
    parent.refresh(logicalNode);
    ConfigurationManager fcfg = getCreateFlowConfigMan(oldflow);
    FlumeConfigData fcd = parent.getConfig(logicalNode);
    try {
      fcfg.setConfig(logicalNode, fcd.flowID, fcd.sourceConfig, fcd.sinkConfig);
    } catch (FlumeSpecException e) {
      throw new IOException(e.getMessage(), e);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfigData

com.cloudera.flume.agent.DirectMasterRPC

com.cloudera.flume.agent.diskfailover.TestDiskFailoverBehavior

com.cloudera.flume.agent.LivenessManager

com.cloudera.flume.agent.LogicalNode

com.cloudera.flume.agent.LogicalNodeManager

com.cloudera.flume.agent.TestAvroMultiMasterRPC$MockAvroServer

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestLogicalNode

com.cloudera.flume.agent.TestRPCMechanisms

com.cloudera.flume.agent.TestRPCMechanisms$MockAvroServer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.