Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    ServletCollector collector2 = new ServletCollector(conf);
    Server collector1_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+1, collector1);
    Server collector2_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+2, collector2);
    Thread.sleep(2000); //for collectors to start
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    HttpConnector conn = new HttpConnector(agent);
    RetryListOfCollectors clist = new RetryListOfCollectors(conf);
    clist.add("http://localhost:"+(PORTNO+1)+"/");
    clist.add("http://localhost:"+(PORTNO+2)+"/");
    conn.setCollectors(clist);
    conn.start();
    //FIXME: somehow need to clue in commit checker which paths to check.
    //       Somehow need

    String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ TestDelayedAcks.SEND_RATE + " 12345 0");
    assertTrue("constSend".equals(resp));
    Thread.sleep(10 * 1000);
    collector1_s.stop();
    Thread.sleep(10 * 1000);
    SeqFileWriter.ENABLE_ROTATION_ON_CLOSE = true;

    String[] stat = agent.getAdaptorList().get("constSend").split(" ");
    long bytesCommitted = Long.valueOf(stat[stat.length -1]);
    assertTrue(bytesCommitted > 0);
    agent.shutdown();
    conn.shutdown();
    Thread.sleep(2000); //for collectors to shut down
    collector2_s.stop();
    Thread.sleep(2000); //for collectors to shut down
   
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
    conf.setInt(AdaptorResetThread.TIMEOUT_OPT, 1000);
    ServletCollector collector = new ServletCollector(conf);
    Server collectorServ = TestDelayedAcks.startCollectorOnPort(conf, PORTNO, collector);
    Thread.sleep(1000);
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    HttpConnector conn = new HttpConnector(agent, "http://localhost:"+PORTNO+"/");
    conn.start();
    String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ SEND_RATE + " 0");
    assertTrue("constSend".equals(resp));
    Thread.sleep(TEST_DURATION_SECS * 1000);
   
    AsyncAckSender sender = (AsyncAckSender)conn.getSender();
    int resets = sender.adaptorReset.getResetCount();
    System.out.println(resets + " resets");
    assertTrue(resets > 0);
   
    agent.shutdown();
    collectorServ.stop();
    conn.shutdown();
    Thread.sleep(5000); //for collector to shut down
   
    long dups = TestFailedCollectorAck.checkDirs(conf, conf.get(SeqFileWriter.OUTPUT_DIR_OPT));
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    conf.setInt(HttpConnector.MIN_POST_INTERVAL_OPT, 100);
    conf.setInt("constAdaptor.sleepVariance", 1);
    conf.setInt("constAdaptor.minSleep", 50);
   
    conf.setInt("chukwaAgent.control.port", 0);
    ChukwaAgent agent = new ChukwaAgent(conf);
    RetryListOfCollectors clist = new RetryListOfCollectors(conf);
    clist.add("http://localhost:"+PORTNO+"/chukwa");
    HttpConnector conn = new HttpConnector(agent);
    conn.setCollectors(clist);
    conn.start();
    Server server = new Server(PORTNO);
    Context root = new Context(server, "/", Context.SESSIONS);

    root.addServlet(new ServletHolder(new ServletCollector(conf)), "/*");
    server.start();
    server.setStopAtShutdown(false);
    Thread.sleep(1000);
    agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ SEND_RATE + " 0");
    assertNotNull(agent.getAdaptor("constSend"));
    Thread.sleep(TEST_DURATION_SECS * 1000);

    String[] stat = agent.getAdaptorList().get("constSend").split(" ");
    long kbytesPerSec = Long.valueOf(stat[stat.length -1]) / TEST_DURATION_SECS / 1000;
    System.out.println("data rate was " + kbytesPerSec + " kb /second");
    assertTrue(kbytesPerSec < WRITE_RATE_KB); //write rate should throttle sends
    assertTrue(kbytesPerSec > MIN_ACCEPTABLE_PERCENT* WRITE_RATE_KB / 100);//an assumption, but should hold true
    agent.shutdown();
  }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  }
 
  public void testOnce()  throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
   
    ChukwaAgent agent = new ChukwaAgent(conf);
   
    assertEquals(0, agent.adaptorCount());

    agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    Chunk c = chunks.waitForAChunk(5000);
    assertNotNull(c);
    String dat = new String(c.getData());
    assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
    assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
    assertTrue(c.getDataType().equals("raw"));
    agent.shutdown();
  }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
  public void testRepeatedly() throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
    int tests = 10; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks

    ChukwaAgent agent = new ChukwaAgent(conf);
    for(int i=0; i < tests; ++i) {
      if(i % 100 == 0)
        System.out.println("buzzed " + i + " times");
     
      assertEquals(0, agent.adaptorCount());
      agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
      assertEquals(1, agent.adaptorCount());
      Chunk c = chunks.waitForAChunk(5000);
      assertNotNull(c);
      String dat = new String(c.getData());
      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
      assertTrue(c.getDataType().equals("raw"));
      if(agent.adaptorCount() > 0)
        agent.stopAdaptor("test", false);
    }
    agent.shutdown();
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  public void testCrSepAdaptor() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
   
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    ChukwaAgent agent = new ChukwaAgent(conf);
    File testFile = makeTestFile("chukwaTest", 80);
    String adaptorId = agent
        .processAddCommand("add test = org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
            + " lines " + testFile + " 0");
    assertTrue(adaptorId.equals("test"));
    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    assertTrue(c.getSeqID() == testFile.length());

    assertTrue(c.getRecordOffsets().length == 80);
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  }

  public void testCrSepAdaptor() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    ChukwaAgent agent = new ChukwaAgent(conf);
    // Remove any adaptor left over from previous run

    // sleep for some time to make sure we don't get chunk from existing streams
    Thread.sleep(5000);
    assertEquals(0, agent.adaptorCount());
    String adaptorId = agent
        .processAddCommand("add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
            + " lines " + testFile + " 0");
    assertNotNull(adaptorId);
    assertEquals(1, agent.adaptorCount());

    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    assertTrue(c.getSeqID() == testFile.length());
    assertTrue(c.getRecordOffsets().length == 80);
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
  public void testRepeatedlyOnBigFile() throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
    int tests = 10; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks

    ChukwaAgent agent = new ChukwaAgent(conf);
    for(int i=0; i < tests; ++i) {
      if(i % 100 == 0)
        System.out.println("buzzed " + i + " times");
     
      assertEquals(0, agent.adaptorCount());
      agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
      assertEquals(1, agent.adaptorCount());
      Chunk c = chunks.waitForAChunk();
      String dat = new String(c.getData());
      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
      assertTrue(c.getDataType().equals("raw"));
      if(agent.adaptorCount() > 0)
        agent.stopAdaptor("test", false);
    }
    agent.shutdown();
  }
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
  }
 
  public void testOffsetInAdaptorName() throws IOException, ChukwaAgent.AlreadyRunningException,
  InterruptedException{
    File testFile = makeTestFile("foo", 120);
    ChukwaAgent agent = new ChukwaAgent(conf);
    assertEquals(0, agent.adaptorCount());
    agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    Thread.sleep(2000);
    agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    chunks.clear();
    agent.shutdown();
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
 
  public FileAdaptorTailer() {
   
    if (conf == null) {
      ChukwaAgent agent = ChukwaAgent.getAgent();
      if (agent != null) {
        conf = agent.getConfiguration();
        if (conf != null) {
          SAMPLE_PERIOD_MS = conf.getInt(
              "chukwaAgent.adaptor.context.switch.time",
              DEFAULT_SAMPLE_PERIOD_MS);
        }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

  • edu.berkeley.chukwa_xtrace.TestXtrAdaptor
  • edu.berkeley.chukwa_xtrace.XtrTestDataGenerator
  • org.apache.hadoop.chukwa.analysis.salsa.fsm.TestFSMBuilder
  • org.apache.hadoop.chukwa.database.TestDatabaseIostat
  • org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.FileAdaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.FileAdaptorTailer
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.FileTailer
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.FileTailingAdaptor
  • org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.TestCharFileTailingAdaptorUTF8
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.