Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask


    chunks.start();
  }

  public void testLogRotate() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    ChukwaAgent agent = new ChukwaAgent();
    // Remove any adaptor left over from previous run
    ChukwaConfiguration cc = new ChukwaConfiguration();
    int portno = cc.getInt("chukwaAgent.control.port", 9093);
    ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
    cli.removeAll();
    // sleep for some time to make sure we don't get chunk from existing streams
    Thread.sleep(5000);
    File testFile = makeTestFile("chukwaLogRotateTest", 80);
    String adaptorId = agent
        .processAddCommand("add lr =org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
            + " lines " + testFile + " 0");
    assertTrue(adaptorId.equals("lr"));
    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    assertTrue(c.getSeqID() == testFile.length());
    assertTrue(c.getRecordOffsets().length == 80);
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      System.out.println("record " + rec + " was: " + record);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    testFile = makeTestFile("chukwaLogRotateTest", 40);
    c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    // assertTrue(c.getSeqID() == testFile.length());
    assertTrue(c.getRecordOffsets().length == 40);
    recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      System.out.println("record " + rec + " was: " + record);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here


  }

  public void testCrSepAdaptor() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    ChukwaAgent agent = new ChukwaAgent();
    // Remove any adaptor left over from previous run

    // sleep for some time to make sure we don't get chunk from existing streams
    Thread.sleep(5000);
    assertEquals(0, agent.adaptorCount());
    String adaptorId = agent
        .processAddCommand("add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
            + " lines " + testFile + " 0");
    assertNotNull(adaptorId);
    assertEquals(1, agent.adaptorCount());

    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    assertTrue(c.getSeqID() == testFile.length());
    assertTrue(c.getRecordOffsets().length == 80);
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here

 
  public void testRepeatedlyOnBigFile() throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
    int tests = 10; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks

    ChukwaAgent agent = new ChukwaAgent(conf);
    for(int i=0; i < tests; ++i) {
      if(i % 100 == 0)
        System.out.println("buzzed " + i + " times");
     
      assertEquals(0, agent.adaptorCount());
      agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
      assertEquals(1, agent.adaptorCount());
      Chunk c = chunks.waitForAChunk();
      String dat = new String(c.getData());
      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
      assertTrue(c.getDataType().equals("raw"));
      if(agent.adaptorCount() > 0)
        agent.stopAdaptor("test", false);
    }
    agent.shutdown();
  }
View Full Code Here

  }
 
  public void testOffsetInAdaptorName() throws IOException, ChukwaAgent.AlreadyRunningException,
  InterruptedException{
    File testFile = makeTestFile("foo", 120);
    ChukwaAgent agent = new ChukwaAgent(conf);
    assertEquals(0, agent.adaptorCount());
    agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    Thread.sleep(2000);
    agent.processAddCommand("add test = filetailer.FileTailingAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    chunks.clear();
    agent.shutdown();
  }
View Full Code Here

  public void startAtOffset() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    conf.setInt("chukwaAgent.adaptor.context.switch.time", 100);
    ChukwaAgent agent = new ChukwaAgent(conf);
    File testFile = makeTestFile();
    int startOffset = 0; // skip first line
    String adaptorId = agent
       .processAddCommand("add org.apache.hadoop.chukwa.datacollection.adaptor"+
            "filetailer.CharFileTailingAdaptorUTF8 "
            + "lines " + startOffset + " " + testFile + " " + startOffset);
    assertTrue(adaptorId != null);
    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    assertTrue(c.getSeqID() == testFile.length() + startOffset);
    System.out.println("RecordOffsets length:" + c.getRecordOffsets().length);
    assertTrue(c.getRecordOffsets().length == 80); // 80 lines in the file.
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      System.out.println("record " + rec + " was: " + record);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
  }
View Full Code Here

  public void testStartAfterOffset() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    conf.setInt("chukwaAgent.adaptor.context.switch.time", 100);
    ChukwaAgent agent = new ChukwaAgent(conf);
    File testFile = makeTestFile();
    int startOffset = 0;
    String adaptorId = agent
        .processAddCommand("add org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8 "
            + "lines "
            + startOffset
            + " "
            + testFile
            + " "
            + (startOffset + 29));
    assertTrue(adaptorId != null);
    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    System.out.println("got chunk");
    while (!c.getDataType().equals("lines")) {
      c = chunks.waitForAChunk();
    }
    assertTrue(c.getSeqID() == testFile.length() + startOffset);

    assertTrue(c.getRecordOffsets().length == 79);// 80 lines in test file,
                                                  // minus the one we skipped
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      System.out.println("record " + rec + " was: " + record);
      assertTrue(record.equals((rec + 1) + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
  }
View Full Code Here

    // Remove any adaptor left over from previous run
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    conf.setInt("chukwaAgent.adaptor.context.switch.time", 100);
    ChukwaAgent agent = new ChukwaAgent(conf);

    File testFile = makeTestFile("chukwaRawTest", 80);
    String adaptorId = agent
        .processAddCommand("add org.apache.hadoop.chukwa.datacollection.adaptor."
            +"filetailer." + name
            + " raw " + testFile + " 0");
    assertNotNull(adaptorId);
    Chunk c = chunks.waitForAChunk(1000);
    assertNotNull(c);
    assertEquals(testFile.length(), c.getData().length);
    assertTrue(c.getDataType().equals("raw"));
    assertTrue(c.getRecordOffsets().length == 1);
    assertTrue(c.getSeqID() == testFile.length());
   
    c = chunks.waitForAChunk(1000);
    assertNull(c);
   
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
  }
View Full Code Here

 
  public FileAdaptorTailer() {
   
    if (conf == null) {
      ChukwaAgent agent = ChukwaAgent.getAgent();
      if (agent != null) {
        conf = agent.getConfiguration();
        if (conf != null) {
          SAMPLE_PERIOD_MS = conf.getInt(
              "chukwaAgent.adaptor.context.switch.time",
              DEFAULT_SAMPLE_PERIOD_MS);
        }
View Full Code Here

  private static Configuration conf = null;
  public static final int MAX_SAMPLE_PERIOD = 60 * 1000;

  FileTailer() {
    if (conf == null) {
      ChukwaAgent agent = ChukwaAgent.getAgent();
      if (agent != null) {
        conf = agent.getConfiguration();
        if (conf != null) {
          SAMPLE_PERIOD_MS = conf.getInt(
              "chukwaAgent.adaptor.context.switch.time",
              DEFAULT_SAMPLE_PERIOD_MS);
        }
View Full Code Here


  public void testStopAndStart() {

    try {
      ChukwaAgent agent = new ChukwaAgent();
      ConsoleOutConnector conn = new ConsoleOutConnector(agent, true);
      conn.start();
     
      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
     
      for(int i=1; i < 20; ++i) {
        cli.add("org.apache.hadoop.chukwa.util.ConstRateAdaptor", "raw" + i, "20000", 0);
        assertTrue(agent.adaptorCount() == 1);
        Thread.sleep(2000);  
        cli.removeAll();
        assertTrue(agent.adaptorCount() == 0);
      }
      agent.shutdown();
      conn.shutdown();
    } catch(Exception e) {
      e.printStackTrace();
      fail(e.toString());
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.