Examples of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.

   */


  @Test(expected = IOException.class)
  public void testDefaultCodec() throws IOException, InterruptedException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "DefaultCodec");


    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath()

View Full Code Here

    shouldSub = Event.containsTag(path);
  }


  protected Writer openWriter(String p) throws IOException {
    LOG.info("Opening " + p);
    FlumeConfiguration conf = FlumeConfiguration.get();


    Path dstPath = new Path(p);
    FileSystem hdfs = dstPath.getFileSystem(conf);


    Writer w = SequenceFile.createWriter(hdfs, conf, dstPath,

View Full Code Here

      }
    };
  }


  public static void main(String argv[]) {
    FlumeConfiguration conf = FlumeConfiguration.get();
    ThriftRawEventSink sink =
        new ThriftRawEventSink("localhost", conf.getCollectorPort());
    try {
      sink.open();


      for (int i = 0; i < 100; i++) {
        Event e = new EventImpl(("This is a test " + i).getBytes());

View Full Code Here

    writer = null;
  }


  @Override
  public void open() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem hdfs;


    // use v0.9.1 compression settings
    if (conf.getCollectorDfsCompressGzipStatus()) {
      LOG.warn("Config property "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP
          + " is deprecated, please use "
          + FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC
          + " set to GzipCodec instead");
      CompressionCodec gzipC = new GzipCodec();
      Compressor gzCmp = gzipC.createCompressor();
      dstPath = new Path(path + gzipC.getDefaultExtension());
      hdfs = dstPath.getFileSystem(conf);
      writer = hdfs.create(dstPath);
      writer = gzipC.createOutputStream(writer, gzCmp);
      LOG.info("Creating HDFS gzip compressed file: " + dstPath.toString());
      return;
    }


    String codecName = conf.getCollectorDfsCompressCodec();
    List<Class<? extends CompressionCodec>> codecs = CompressionCodecFactory
        .getCodecClasses(FlumeConfiguration.get());
    CompressionCodec codec = null;
    ArrayList<String> codecStrs = new ArrayList<String>();
    codecStrs.add("None");

View Full Code Here

  /**
   * Test that full init, create client, shutdown lifecycle works correctly
   */
  @Test
  public void testLifecycle() throws IOException, InterruptedException, KeeperException {
    FlumeConfiguration cfg = FlumeConfiguration.get();
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    ZooKeeperService svc = new ZooKeeperService();
    svc.init(cfg);
    
    Assert.assertTrue("ZooKeeperService did not initialise", svc.isInitialised());

View Full Code Here

  /**
   * Test that we have correctly disabled the maxClientCnxns bound
   */
  @Test
  public void testMultipleClients() throws IOException, InterruptedException, KeeperException {
    FlumeConfiguration cfg = FlumeConfiguration.get();
    File tmp = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.MASTER_ZK_SERVERS, "localhost:2181:3181:4181");
    cfg.set(FlumeConfiguration.MASTER_ZK_LOGDIR, tmp.getAbsolutePath());
    ZooKeeperService svc = new ZooKeeperService();
    svc.init(cfg);
    
    Assert.assertTrue("ZooKeeperService did not initialise", svc.isInitialised());

View Full Code Here

      throw new IOException("Waiting for queue element was interrupted! " + e);
    }
  }


  public static void main(String[] argv) {
    FlumeConfiguration conf = FlumeConfiguration.get();
    PrioritizedThriftEventSource src = new PrioritizedThriftEventSource(conf
        .getCollectorPort());


    try {
      src.open();
      Event e;

View Full Code Here

      System.exit(-1);
    }


    String[] args = argv;


    FlumeConfiguration conf = FlumeConfiguration.hardExitLoadConfig();
    int maxTriesPerMin = conf.getMaxRestartsPerMin();


    Watchdog watchdog = new Watchdog(args);
    watchdog.run(maxTriesPerMin);
  }

View Full Code Here

      }
      specman.stop();


      reaper.interrupt();


      FlumeConfiguration cfg = FlumeConfiguration.get();
      if (cfg.getMasterStore().equals(ZK_CFG_STORE)) {
        ZooKeeperService.get().shutdown();
      }


    } catch (IOException e) {
      LOG.error("Exception when shutting down master!", e);

View Full Code Here

    String interactiveprop = System.getProperty("fwdstdin");
    boolean interactive = (interactiveprop != null);


    String[] args = argv;


    FlumeConfiguration conf = FlumeConfiguration.hardExitLoadConfig();
    int maxTriesPerMin = conf.getMaxRestartsPerMin();


    Watchdog watchdog = new Watchdog(args, interactive);
    watchdog.run(maxTriesPerMin);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.FaultyFlumeLocalAgentOOME

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestAgentSink

com.cloudera.flume.agent.TestAvroMultiMasterRPC

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestRPCMechanisms

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.