Package org.apache.hadoop.chukwa.conf

Examples of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

49
50
51
52
53
54
55
56
57
58
59
60
61
 
  static Logger log = Logger.getLogger(SinkArchiver.class);
 
  public static void main(String[] args) {
    try {
      Configuration conf = new ChukwaConfiguration();
      if(conf.get(ChukwaArchiveDataTypeOutputFormat.GROUP_BY_CLUSTER_OPTION_NAME) == null )
        conf.set(ChukwaArchiveDataTypeOutputFormat.GROUP_BY_CLUSTER_OPTION_NAME, "true");
      FileSystem fs = FileSystem.get(conf);
      SinkArchiver archiver = new SinkArchiver();
      archiver.exec(fs, conf);   
    } catch(IOException e) {
      e.printStackTrace();
View Full Code Here
138
139
140
141
142
143
144
145
146
    JobClient.runJob(jobConf);
    return 0;
  }

  public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new ChukwaConfiguration(), new ChukwaArchiveBuilder(),
        args);
    System.exit(res);
  }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
  protected void setUp() throws Exception {
    MockTriggerAction.reset();
  }

  public void testSuccessTrigger() throws Exception {
    ChukwaConfiguration conf = new ChukwaConfiguration();
    conf.set(CHUKWA_CONSTANT.POST_DEMUX_SUCCESS_ACTION,
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction");

    PostProcessorManager postProcessManager = new PostProcessorManager(conf);
    assertTrue("processPostMoveTriggers returned false",
            postProcessManager.processPostMoveTriggers(SAMPLE_PATHS));
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
    assertEquals("Incorrect Trigger event found", TriggerEvent.POST_DEMUX_SUCCESS,
            events.next());
  }

  public void testMultiSuccessTrigger() throws Exception {
    ChukwaConfiguration conf = new ChukwaConfiguration();
    conf.set(CHUKWA_CONSTANT.POST_DEMUX_SUCCESS_ACTION,
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction," +
            "org.apache.hadoop.chukwa.extraction.demux.MockTriggerAction");

    PostProcessorManager postProcessManager = new PostProcessorManager(conf);
    assertTrue("processPostMoveTriggers returned false",
View Full Code Here
281
282
283
284
285
286
287
288
289
290
291
      System.setProperty("hadoop.log.dir", System.getProperty(
          "test.build.data", "/tmp"));
   
      String[] sortArgs = { DEMUX_INPUT_PATH.toString(), DEMUX_OUTPUT_PATH.toString() };
      //      JobConf job = mr.createJobConf();
      JobConf job = new JobConf(new ChukwaConfiguration(), Demux.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-Demux_" + day.format(new Date()));
      job.setInputFormat(SequenceFileInputFormat.class);
      job.setMapperClass(Demux.MapClass.class);
      job.setPartitionerClass(ChukwaRecordPartitioner.class);
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
    // Test FSMBuilder (job history only)
    log.info("Testing FSMBuilder (Job History only)");
    System.out.println("In JobHistory020");
    // Run FSMBuilder on Demux output
    try {
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(JobHistoryTaskDataMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
View Full Code Here
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
    log.info("Testing FSMBuilder (ClientTrace only)");
    System.out.println("In ClientTrace020");
    // Run FSMBuilder on Demux output
    try {
      // Process TaskTracker shuffle clienttrace entries first
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(TaskTrackerClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
      job.setMapOutputValueClass(FSMIntermedEntry.class);
      job.setMapOutputKeyClass(ChukwaRecordKey.class);

      job.setInputFormat(SequenceFileInputFormat.class);
      job.setOutputKeyClass(ChukwaRecordKey.class);
      job.setOutputValueClass(ChukwaRecord.class);
      job.setOutputFormat(ChukwaRecordOutputFormat.class);
      job.setNumReduceTasks(1);

      Path inputPath = new Path(DEMUX_OUTPUT_PATH.toString()+File.separator+"/*/*/ClientTraceDetailed*.evt");
      Path fsmOutputPath1 = new Path(fileSys.getUri().toString()+File.separator+fsmSink+"1");

      FileInputFormat.setInputPaths(job, inputPath);
      FileOutputFormat.setOutputPath(job, fsmOutputPath1);

      String[] jars = new File(System.getenv("CHUKWA_HOME")).list(new FilenameFilter() {
        public boolean accept(File dir, String name) {
          return name.endsWith(".jar");
        }
      });
      job.setJar(System.getenv("CHUKWA_HOME")+File.separator+jars[0]);
      JobClient.runJob(job);
      System.out.println("Processed TaskTracker ClientTrace.");

      // Process DataNode clienttrace entries
      job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(DataNodeClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
      boolean failed = false;
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:1234/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      File tmpOutput = TempFileUtil.makeBinary(2000);

      cli.addFile("unknown", tmpOutput.getAbsolutePath());
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  private byte[] test = Bytes.toBytes("1234567890 Key Value");
  private ChukwaConfiguration cc;
  long timestamp = 1234567890;
 
  public TestHBaseWriter() {
    cc = new ChukwaConfiguration();

    conf = HBaseConfiguration.create();
    conf.set("hbase.hregion.memstore.flush.size", String.valueOf(128*1024));
    try {
      util = new HBaseTestingUtility(conf);
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
  /**
   * Standard workflow
   */
  public void testScenario1() {
    ChukwaConfiguration cc = new ChukwaConfiguration();
    String tempDirectory = System.getProperty("test.build.data", "/tmp");
    String chukwaRootDir = tempDirectory + "/demuxManagerTest_" + System.currentTimeMillis() +"/";
   
    cc.set(CHUKWA_CONSTANT.HDFS_DEFAULT_NAME_FIELD, "file:///");
    cc.set(CHUKWA_CONSTANT.CHUKWA_ROOT_DIR_FIELD, chukwaRootDir );
    cc.set(CHUKWA_CONSTANT.CHUKWA_ARCHIVE_DIR_FIELD, chukwaRootDir +"/archives/" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_POST_PROCESS_DIR_FIELD, chukwaRootDir +"/postProcess" );
    cc.set(CHUKWA_CONSTANT.CHUKWA_DATA_SINK_DIR_FIELD, chukwaRootDir +"/logs" );
   
    
    try {
     
      File dataSinkDirectory = new File(chukwaRootDir +"/logs");
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.conf.ChukwaConfiguration

  • hitune.analysis.mapreduce.Analysis
  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.database.TestDatabaseAggregator
  • org.apache.hadoop.chukwa.database.TestDatabaseDataExpiration
  • org.apache.hadoop.chukwa.database.TestDatabaseIostat
  • org.apache.hadoop.chukwa.database.TestDatabaseTableCreator
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileExpirationPolicy
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailer
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorBigRecord
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestFileTailingAdaptorPreserveLines
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.