Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask


    conf.setInt(AdaptorResetThread.TIMEOUT_OPT, 1000);
    ServletCollector collector = new ServletCollector(conf);
    Server collectorServ = TestDelayedAcks.startCollectorOnPort(conf, PORTNO, collector);
    Thread.sleep(1000);
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    HttpConnector conn = new HttpConnector(agent, "http://localhost:"+PORTNO+"/");
    conn.start();
    String resp = agent.processAddCommand("add constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ SEND_RATE + " 0");
    assertTrue("adaptor_constSend".equals(resp));
    Thread.sleep(TEST_DURATION_SECS * 1000);
   
    AsyncAckSender sender = (AsyncAckSender)conn.getSender();
    int resets = sender.adaptorReset.getResetCount();
    System.out.println(resets + " resets");
    assertTrue(resets > 0);
   
    agent.shutdown();
    collectorServ.stop();
    conn.shutdown();
    Thread.sleep(5000); //for collector to shut down
   
    long dups = TestFailedCollectorAck.checkDirs(conf, conf.get(SeqFileWriter.OUTPUT_DIR_OPT));
View Full Code Here


    Configuration conf = new Configuration();
    File baseDir = new File(System.getProperty("test.build.data", "/tmp"));
    conf.set("chukwaAgent.checkpoint.dir", baseDir.getCanonicalPath());
    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
    conf.set("chukwaAgent.control.port", "0");
    ChukwaAgent agent = new ChukwaAgent(conf);
    ChunkCatcherConnector chunks = new ChunkCatcherConnector();
    chunks.start();

    System.setProperty("xtrace.reporter", "edu.berkeley.xtrace.reporting.TcpReporter");
    System.setProperty("xtrace.tcpdest", "localhost:7831");

    assertEquals(0, agent.adaptorCount());
    agent.processAddCommand("add edu.berkeley.chukwa_xtrace.XtrAdaptor XTrace TcpReportSource 0");
    assertEquals(1, agent.adaptorCount());
   
    XTraceContext.startTrace("test", "testtrace", "atag");
    XTraceContext.logEvent("test", "label");
    Chunk c = chunks.waitForAChunk();
    String report = new String(c.getData());
    assertTrue(report.contains("Agent: test"));
    assertTrue(report.contains("Tag: atag"));
    System.out.println(report);
    System.out.println("-- next chunk --- ");

    c = chunks.waitForAChunk();
    report = new String(c.getData());
    assertTrue(report.contains("Agent: test"));
    assertTrue(report.contains("Label: label"));
    System.out.println(report);

    System.out.println("OK");
    agent.shutdown();
  }
View Full Code Here

 
 
  public void testOnce()  throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
   
    ChukwaAgent agent = new ChukwaAgent(conf);
   
    assertEquals(0, agent.adaptorCount());

    String name =agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
    assertEquals(1, agent.adaptorCount());
    assertEquals(name, "adaptor_test");
    Chunk c = chunks.waitForAChunk(5000);
    assertNotNull(c);
    String dat = new String(c.getData());
    assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
    assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
    assertTrue(c.getDataType().equals("raw"));
    agent.shutdown();
  }
View Full Code Here

 
  public void testRepeatedly() throws IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
    int tests = 10; //SHOULD SET HIGHER AND WATCH WITH lsof to find leaks

    ChukwaAgent agent = new ChukwaAgent(conf);
    for(int i=0; i < tests; ++i) {
      if(i % 100 == 0)
        System.out.println("buzzed " + i + " times");
     
      assertEquals(0, agent.adaptorCount());
      String name = agent.processAddCommand("add test = FileAdaptor raw " +testFile.getCanonicalPath() + " 0");
      assertEquals(1, agent.adaptorCount());
      assertEquals(name, "adaptor_test");
      Chunk c = chunks.waitForAChunk(5000);
      assertNotNull(c);
      String dat = new String(c.getData());
      assertTrue(dat.startsWith("0 abcdefghijklmnopqrstuvwxyz"));
      assertTrue(dat.endsWith("9 abcdefghijklmnopqrstuvwxyz\n"));
      assertTrue(c.getDataType().equals("raw"));
      while(agent.adaptorCount() > 0) {
        agent.stopAdaptor("adaptor_test", false);
        Thread.sleep(1000);
      }
    }
    agent.shutdown();
  }
View Full Code Here

  public void testWithPs() throws ChukwaAgent.AlreadyRunningException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
    ChukwaAgent agent = new ChukwaAgent(conf);
    ChunkCatcherConnector chunks = new ChunkCatcherConnector();
    chunks.start();
    String psAgentID = agent.processAddCommand(
        "add exec= org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor ps 500 ps aux 0");
    Chunk c = chunks.waitForAChunk();
    System.out.println(new String(c.getData()));
    assertNotNull(psAgentID);
    agent.shutdown();
  }
View Full Code Here

   */
  public void testForLeaks()  throws ChukwaAgent.AlreadyRunningException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
    ChukwaAgent agent = new ChukwaAgent(conf);

    chunks = new ConsoleOutConnector(agent, false);
    chunks.start();
    assertEquals(0, agent.adaptorCount());
    String lsID = agent.processAddCommand(
      "add exec= org.apache.hadoop.chukwa.datacollection.adaptor.ExecAdaptor Listing 100 /bin/sleep 1 0");
    Thread.sleep( 5*1000); //RAISE THIS to test longer
    System.out.println("stopped ok");
  }
View Full Code Here

    conf.set("chukwaAgent.checkpoint.dir", checkpointDir.getCanonicalPath());
    conf.set("chukwaAgent.checkpoint.name", "checkpoint_");
    conf.setInt("chukwaAgent.control.port", 0);
    conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
   
    agent = new ChukwaAgent(conf);
    File emptyDir = new File(baseDir, "emptyDir2");
    createEmptyDir(emptyDir);
   
    assertEquals(0, agent.adaptorCount());
    //check filtering with empty directory
    agent.processAddCommand("add emptydir2= DirTailingAdaptor raw " + emptyDir + " *file filetailer.CharFileTailingAdaptorUTF8 0");
    assertEquals(1, agent.adaptorCount());

    File dirWithFile = new File(baseDir, "dir3");
    dirWithFile.delete();
    assertFalse("temp directory not empty",dirWithFile.exists());
     
    //this file should be found by the filter
    dirWithFile.mkdir();
    File inDir = File.createTempFile("atemp", "file", dirWithFile);
    inDir.deleteOnExit();
    //This file should not be found by the filter
    File noreadFile = File.createTempFile("atemp", "noread", dirWithFile);
    noreadFile.deleteOnExit();
   
    //apply filter *file
    agent.processAddCommand("add dir3= DirTailingAdaptor raw " + dirWithFile + " *file filetailer.CharFileTailingAdaptorUTF8 0");
    Thread.sleep(3000);
    assertEquals(3, agent.adaptorCount());
    agent.shutdown();

    conf.setBoolean("chukwaAgent.checkpoint.enabled", true);
    Thread.sleep(1500); //wait a little bit to make sure new file ts is > last checkpoint time.
    File anOldFile = File.createTempFile("oldXYZ","file", dirWithFile);
    File aNewFile = File.createTempFile("new", "file", dirWithFile);
    anOldFile.deleteOnExit();
    aNewFile.deleteOnExit();
    anOldFile.setLastModified(10);//just after epoch
    agent = new ChukwaAgent(conf); //restart agent.
   
   Thread.sleep(3 * SCAN_INTERVAL); //wait a bit for the new file to be detected.
   assertTrue(aNewFile.exists());
  
    //make sure we started tailing the new, not the old, file.
View Full Code Here

  
    boolean retry = true;
    while(retry) {
      try {
        retry = false;
        agent = new ChukwaAgent(conf);
      } catch(Exception e) {
        retry = true;
      }
    }
    File emptyDir = new File(baseDir, "emptyDir");
    createEmptyDir(emptyDir);
   
    assertEquals(0, agent.adaptorCount());
    agent.processAddCommand("add emptydir= DirTailingAdaptor raw " + emptyDir + " filetailer.CharFileTailingAdaptorUTF8 0");
    assertEquals(1, agent.adaptorCount());

    File dirWithFile = new File(baseDir, "dir2");
    dirWithFile.delete();
    assertFalse("temp directory not empty",dirWithFile.exists());
     
    dirWithFile.mkdir();
    File inDir = File.createTempFile("atemp", "file", dirWithFile);
    inDir.deleteOnExit();
    agent.processAddCommand("add dir2= DirTailingAdaptor raw " + dirWithFile + " *file filetailer.CharFileTailingAdaptorUTF8 0");
    Thread.sleep(3000);
    assertEquals(3, agent.adaptorCount());
    System.out.println("DirTailingAdaptor looks OK before restart");
    agent.shutdown();

    conf.setBoolean("chukwaAgent.checkpoint.enabled", true);
    Thread.sleep(1500); //wait a little bit to make sure new file ts is > last checkpoint time.
    File anOldFile = File.createTempFile("oldXYZ","file", dirWithFile);
    File aNewFile = File.createTempFile("new", "file", dirWithFile);
    anOldFile.deleteOnExit();
    aNewFile.deleteOnExit();
    anOldFile.setLastModified(10);//just after epoch
    agent = new ChukwaAgent(conf); //restart agent.
   
   Thread.sleep(3 * SCAN_INTERVAL); //wait a bit for the new file to be detected.
   assertTrue(aNewFile.exists());
  
    //make sure we started tailing the new, not the old, file.
View Full Code Here

  //we delete the file and also change the data type each time through the loop
  //to make sure we get the cached chunk.
  public void resendAfterStop(String adaptorthrows IOException,
  ChukwaAgent.AlreadyRunningException, InterruptedException {
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    String ADAPTORID = "adaptor_test" + System.currentTimeMillis();
    String STR = "test data";
    int PORTNO = 9878;
    DatagramSocket send = new DatagramSocket();
    byte[] buf = STR.getBytes();
    DatagramPacket p = new DatagramPacket(buf, buf.length);
    p.setSocketAddress(new InetSocketAddress("127.0.0.1",PORTNO));
   
    assertEquals(0, agent.adaptorCount());
    String name =agent.processAddCommand("add "+ ADAPTORID + " = "+adaptor+" UDPAdaptor raw "+PORTNO+ " 0");
    assertEquals(name, ADAPTORID);
    Thread.sleep(500);
    send.send(p);
   
    for(int i=0; i< 5; ++i) {
      Chunk c = chunks.waitForAChunk(5000);
      System.out.println("received " + i);
      assertNotNull(c);
      String dat = new String(c.getData());
      assertTrue(dat.equals(STR));
      assertTrue(c.getDataType().equals("raw"));
      assertEquals(c.getSeqID(), STR.length());
     
      agent.stopAdaptor(name, AdaptorShutdownPolicy.RESTARTING);
      Thread.sleep(500); //for socket to deregister
      name =agent.processAddCommand("add "+ADAPTORID + " = "+adaptor+" UDPAdaptor raw "+PORTNO + " 0");
      assertEquals(name, ADAPTORID);
    }
    Chunk c = chunks.waitForAChunk(5000);

    Thread.sleep(500);
   
    buf = "different data".getBytes();
    p = new DatagramPacket(buf, buf.length);  
    p.setSocketAddress(new InetSocketAddress("127.0.0.1",PORTNO));
    send.send(p);
    c = chunks.waitForAChunk(5000);
    assertNotNull(c);
    assertEquals(buf.length + STR.length(), c.getSeqID());
   
    agent.stopAdaptor(name, true);
    assertEquals(0, agent.adaptorCount());
    Thread.sleep(500);//before re-binding
    agent.shutdown();
  }
View Full Code Here

      conf.set("chukwaAgent.tags", "cluster=\"demo\"");
      DataFactory.getInstance().addDefaultTag(conf.get("chukwaAgent.tags", "cluster=\"unknown\""));
      conf.set("chukwaAgent.checkpoint.dir", System.getenv("CHUKWA_DATA_DIR")+File.separator+"tmp");
      conf.set("chukwaAgent.checkpoint.interval", "10000");
      int portno = conf.getInt("chukwaAgent.control.port", agentPort);
      agent = new ChukwaAgent(conf);
      conn = new HttpConnector(agent, "http://localhost:"+collectorPort+"/chukwa");
      conn.start();     
      sender = new ChukwaHttpSender(conf);
      ArrayList<String> collectorList = new ArrayList<String>();
      collectorList.add("http://localhost:"+collectorPort+"/chukwa");
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.