Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask


public class TestFailedCollector extends TestCase {

  public void testFailedCollector() {
    try {
      ChukwaAgent agent = new ChukwaAgent();
      boolean failed = false;
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:1234/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
      int portno = cc.getInt("chukwaAgent.control.port", 9093);
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      File tmpOutput = TempFileUtil.makeBinary(2000);

      cli.addFile("unknown", tmpOutput.getAbsolutePath());
      System.out.println("have " + agent.adaptorCount() + " running adaptors");
      cli.removeFile("unknown", tmpOutput.getAbsolutePath());

      tmpOutput.delete();
      assertFalse(failed);
      System.out.println("done");
      agent.shutdown();
      connector.shutdown();
      Thread.sleep(2000);
    } catch (Exception e) {
      e.printStackTrace();
    }
View Full Code Here


    conf.setInt(HttpConnector.MIN_POST_INTERVAL_OPT, 100);
    conf.setInt("constAdaptor.sleepVariance", 1);
    conf.setInt("constAdaptor.minSleep", 50);
   
    conf.setInt("chukwaAgent.control.port", 0);
    ChukwaAgent agent = new ChukwaAgent(conf);
    RetryListOfCollectors clist = new RetryListOfCollectors(conf);
    clist.add("http://localhost:"+PORTNO+"/chukwa");
    HttpConnector conn = new HttpConnector(agent);
    conn.setCollectors(clist);
    conn.start();
    Server server = new Server(PORTNO);
    Context root = new Context(server, "/", Context.SESSIONS);

    root.addServlet(new ServletHolder(new ServletCollector(conf)), "/*");
    server.start();
    server.setStopAtShutdown(false);
    Thread.sleep(1000);
    agent.processAddCommand("add adaptor_constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ SEND_RATE + " 0");
    assertNotNull(agent.getAdaptor("adaptor_constSend"));
    Thread.sleep(TEST_DURATION_SECS * 1000);

    String[] stat = agent.getAdaptorList().get("adaptor_constSend").split(" ");
    long kbytesPerSec = Long.valueOf(stat[stat.length -1]) / TEST_DURATION_SECS / 1000;
    System.out.println("data rate was " + kbytesPerSec + " kb /second");
    assertTrue(kbytesPerSec < WRITE_RATE_KB); //write rate should throttle sends
    assertTrue(kbytesPerSec > MIN_ACCEPTABLE_PERCENT* WRITE_RATE_KB / 100);//an assumption, but should hold true
    agent.shutdown();
  }
View Full Code Here

    conf.set("chukwaAgent.control.port", "0");
    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
    conf.setInt("chukwaAgent.adaptor.context.switch.time", 500);
    conf.setInt(AdaptorResetThread.TIMEOUT_OPT, ACK_TIMEOUT);

    ChukwaAgent agent = new ChukwaAgent(conf);
    ChunkCatcherConnector chunks = new ChunkCatcherConnector();
    chunks.start();
    assertEquals(0, agent.adaptorCount());
    File testFile = makeTestFile("testDA", 50, new File(System.getProperty("test.build.data", "/tmp")));
    long len = testFile.length();
    System.out.println("wrote data to " + testFile);
    AdaptorResetThread restart = new AdaptorResetThread(conf, agent);
    //start timeout thread
    agent.processAddCommand("add fta = "+ FileTailingAdaptor.class.getCanonicalName()
        + " testdata " + testFile.getCanonicalPath() + " 0" );
   
   
    assertEquals(1, agent.adaptorCount());
    Chunk c1 = chunks.waitForAChunk();
    assertNotNull(c1);
    List<CommitListEntry> pendingAcks = new ArrayList<CommitListEntry>();
    pendingAcks.add(new DelayedCommit(c1.getInitiator(), c1.getSeqID(),
        c1.getData().length, "foo", c1.getSeqID(), agent.getAdaptorName(c1.getInitiator())));
    restart.reportPending(pendingAcks);

    assertEquals(len, c1.getData().length);
    Thread.sleep(ACK_TIMEOUT*2);
    int resetCount = restart.resetTimedOutAdaptors(ACK_TIMEOUT);
    Chunk c2 = chunks.waitForAChunk(1000);
    assertNotNull(c2);
    assertEquals(len, c2.getData().length);
    assertTrue(resetCount > 0);
    agent.shutdown();
   
    testFile.delete();
  }
View Full Code Here

      String outputDirectory = buildConf(conf);
      ServletCollector collector = new ServletCollector(conf);
      Server collectorServ = startCollectorOnPort(conf, PORTNO, collector);
      Thread.sleep(1000);
     
      ChukwaAgent agent = new ChukwaAgent(conf);
      HttpConnector conn = new HttpConnector(agent, "http://localhost:"+PORTNO+"/");
      conn.start();
      String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
          " testData "+ SEND_RATE + " 0");
      assertTrue("adaptor_constSend".equals(resp));
      Thread.sleep(END2END_TEST_SECS * 1000);

      //do the shutdown directly, here, so that acks are still processed.
      assertNotNull(agent.getAdaptor("adaptor_constSend"));
      long bytesOutput = agent.getAdaptor("adaptor_constSend").shutdown(AdaptorShutdownPolicy.GRACEFULLY);
      Thread.sleep(CLIENT_SCANPERIOD + SERVER_SCANPERIOD + ROTATEPERIOD + 3000);
     
      String[] stat = agent.getAdaptorList().get("adaptor_constSend").split(" ");
      long bytesCommitted = Long.valueOf(stat[stat.length -1]);
     
      long bytesPerSec = bytesOutput / (1000 * END2END_TEST_SECS);
      System.out.println("data rate was " + bytesPerSec + " kb /second");
  
      //all data should be committed
      System.out.println(bytesCommitted + " bytes committed");
      System.out.println(bytesOutput + " bytes output");
      System.out.println("difference is " + (bytesOutput - bytesCommitted));
      ChukwaWriter w = collector.getWriter();
      long bytesWritten = ((SeqFileWriter)w).getBytesWritten();
      System.out.println("collector wrote " + bytesWritten);

      assertEquals(bytesCommitted, bytesOutput);
      assertEquals(bytesWritten, bytesCommitted);
      //We need a little imprecision here, since the send rate is a bit bursty,
      //and since some acks got lost after the adaptor was stopped.
      assertTrue(bytesPerSec > 9 * SEND_RATE/ 1000 / 10);
      AsyncAckSender sender = (AsyncAckSender)conn.getSender();
      assertEquals(0, sender.adaptorReset.getResetCount());
     
      agent.shutdown();
      collectorServ.stop();
      conn.shutdown();
      Thread.sleep(5000); //for collector to shut down
      TestDirTailingAdaptor.nukeDirContents(new File(outputDirectory));
     
View Full Code Here

      root.addServlet(new ServletHolder(collector), "/*");
      server.start();
      server.setStopAtShutdown(false);

      Thread.sleep(1000);
      ChukwaAgent agent = new ChukwaAgent();
      HttpConnector connector = new HttpConnector(agent,
          "http://localhost:9990/chukwa");
      connector.start();

      ChukwaConfiguration cc = new ChukwaConfiguration();
View Full Code Here

  MockHttpServletRequest request;
  MockHttpServletResponse response;
  StringBuilder sb;

  protected void setUp() throws Exception {
    agent = new ChukwaAgent();

    ServletHolder servletHolder = new ServletHolder(ServletContainer.class);
    servletHolder.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
            "com.sun.jersey.api.core.PackagesResourceConfig");
    servletHolder.setInitParameter("com.sun.jersey.config.property.packages",
View Full Code Here

  public void testStopAndStart() {

    try {
      Configuration conf = new Configuration();
      conf.setInt("chukwaAgent.control.port", 0);
      ChukwaAgent agent = new ChukwaAgent(conf);
      ConsoleOutConnector conn = new ConsoleOutConnector(agent, true);
      conn.start();

      int portno = agent.getControllerPort();
      ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);

      for (int i = 1; i < 20; ++i) {
        String adaptorId = cli.add(
            "org.apache.hadoop.chukwa.util.ConstRateAdaptor", "raw" + i, "2000"
                + i, 0);
        assertNotNull(adaptorId);
        Thread.sleep(2000);
        cli.removeAll();
      }
      agent.shutdown();
      conn.shutdown();
      Thread.sleep(2000);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.toString());
View Full Code Here

  public void testMultiStopAndStart() {

    try {
      Configuration conf = new Configuration();
      conf.setInt("chukwaAgent.control.port", 0);
      ChukwaAgent agent = new ChukwaAgent(conf);
      ConsoleOutConnector conn = new ConsoleOutConnector(agent, true);
      conn.start();
      int count = agent.adaptorCount();
      for (int trial = 0; trial < 20; ++trial) {
        ArrayList<String> runningAdaptors = new ArrayList<String>();

        for (int i = 1; i < 7; ++i) {
          String l = agent
              .processAddCommand("add  org.apache.hadoop.chukwa.util.ConstRateAdaptor  raw"
                  + i + " 2000" + i + " 0");
          assertTrue(l != null);
          runningAdaptors.add(l);
        }
        Thread.sleep(1000);
        for (String l : runningAdaptors)
          agent.stopAdaptor(l, false);
        Thread.sleep(5000);
        assertTrue(agent.adaptorCount() == count);
      }
      agent.shutdown();
      Thread.sleep(2000);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.toString());
    }
View Full Code Here

      DaemonWatcher.bailout(-1);
    }

   
    List<Chunk> chunks = new LinkedList<Chunk>();
    ChukwaAgent agent = null;// ChukwaAgent.getAgent();
   
    log.info("processing data for QueueToWriterConnector");
   
    while ( isRunning ||  chunkQueue.size() != 0 || chunks.size() != 0) {
      try {
        if (chunks.size() == 0) {
         
          if (isBackfilling && chunkQueue.size() == 0) {
            Thread.sleep(300);
            continue;
          }
          chunkQueue.collect(chunks, MAX_SIZE_PER_POST);
          log.info("Got " + chunks.size() + " chunks back from the queue");
        }      
       
        writer.add(chunks);
       
        if (agent != null) {
          for(Chunk chunk: chunks) {
            agent.reportCommit(chunk.getInitiator(), chunk.getSeqID());
          }
        }
       
        chunks.clear();
       
View Full Code Here

    ServletCollector collector2 = new ServletCollector(conf);
    Server collector1_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+1, collector1);
    Server collector2_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+2, collector2);
    Thread.sleep(2000); //for collectors to start
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    HttpConnector conn = new HttpConnector(agent);
    RetryListOfCollectors clist = new RetryListOfCollectors(conf);
    clist.add("http://localhost:"+(PORTNO+1)+"/");
    clist.add("http://localhost:"+(PORTNO+2)+"/");
    conn.setCollectors(clist);
    conn.start();
    //FIXME: somehow need to clue in commit checker which paths to check.
    //       Somehow need

    String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ TestDelayedAcks.SEND_RATE + " 12345 0");
    assertTrue("constSend".equals(resp));
    Thread.sleep(10 * 1000);
    collector1_s.stop();
    Thread.sleep(10 * 1000);
    SeqFileWriter.ENABLE_ROTATION_ON_CLOSE = true;

    String[] stat = agent.getAdaptorList().get("constSend").split(" ");
    long bytesCommitted = Long.valueOf(stat[stat.length -1]);
    assertTrue(bytesCommitted > 0);
    agent.shutdown();
    conn.shutdown();
    Thread.sleep(2000); //for collectors to shut down
    collector2_s.stop();
    Thread.sleep(2000); //for collectors to shut down
   
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.