Package org.apache.hadoop.chukwa.datacollection.agent

Examples of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask


  public void testCrSepAdaptor() throws IOException, InterruptedException,
      ChukwaAgent.AlreadyRunningException {
   
    Configuration conf = new Configuration();
    conf.set("chukwaAgent.control.port", "0");
    ChukwaAgent agent = new ChukwaAgent(conf);
    File testFile = makeTestFile("chukwaTest", 80,baseDir);
    String adaptorId = agent
        .processAddCommand("add adaptor_test = org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8"
            + " lines " + testFile + " 0");
    assertTrue(adaptorId.equals("adaptor_test"));
    System.out.println("getting a chunk...");
    Chunk c = chunks.waitForAChunk();
    assertTrue(c.getSeqID() == testFile.length());

    assertTrue(c.getRecordOffsets().length == 80);
    int recStart = 0;
    for (int rec = 0; rec < c.getRecordOffsets().length; ++rec) {
      String record = new String(c.getData(), recStart,
          c.getRecordOffsets()[rec] - recStart + 1);
      assertTrue(record.equals(rec + " abcdefghijklmnopqrstuvwxyz\n"));
      recStart = c.getRecordOffsets()[rec] + 1;
    }
    assertTrue(c.getDataType().equals("lines"));
    agent.stopAdaptor(adaptorId, false);
    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here


    pw = new PrintWriter(new FileOutputStream(tmpOutput));
    pw.println("Second");
    pw.close();
   
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    String adaptorID = agent.processAddCommand("add lr = filetailer.RCheckFTAdaptor test " + tmpOutput.getAbsolutePath() + " 0");
    assertNotNull(adaptorID);
   
    Chunk c = chunks.waitForAChunk(2000);
    assertNotNull(c);
    assertTrue(c.getData().length == 6);
    assertTrue("First\n".equals(new String(c.getData())));
    c = chunks.waitForAChunk(2000);
    assertNotNull(c);
    assertTrue(c.getData().length == 7);   
    assertTrue("Second\n".equals(new String(c.getData())));

    pw = new PrintWriter(new FileOutputStream(tmpOutput, true));
    pw.println("Third");
    pw.close();
    c = chunks.waitForAChunk(2000);
   
    assertNotNull(c);
    assertTrue(c.getData().length == 6);   
    assertTrue("Third\n".equals(new String(c.getData())));
    Thread.sleep(1500);
   
    tmpOutput.renameTo(new File(baseDir, "rotateTest.2"));
    pw = new PrintWriter(new FileOutputStream(tmpOutput, true));
    pw.println("Fourth");
    pw.close();
    c = chunks.waitForAChunk(2000);

    assertNotNull(c);
    System.out.println("got " + new String(c.getData()));
    assertTrue("Fourth\n".equals(new String(c.getData())));

    Thread.sleep(1500);
   
    tmpOutput.renameTo(new File(baseDir, "rotateTest.3"));
    Thread.sleep(400);
    pw = new PrintWriter(new FileOutputStream(tmpOutput, true));
    pw.println("Fifth");
    pw.close();
    c = chunks.waitForAChunk(2000);
    assertNotNull(c);
    System.out.println("got " + new String(c.getData()));
    assertTrue("Fifth\n".equals(new String(c.getData())));

    agent.shutdown();
    Thread.sleep(2000);
  }
View Full Code Here

  @Consumes("application/json")
  @Produces({"text/xml","text/plain"})
  public Response addAdaptor(@Context ServletContext context,
                             @QueryParam("viewType") String viewType,
                             String postBody) {
    ChukwaAgent agent = (ChukwaAgent)context.getAttribute("ChukwaAgent");

    if (postBody == null) return badRequestResponse("Empty POST body.");

    // parse the json.
    StringBuilder addCommand = new StringBuilder("add ");
    try {
      JSONObject reqJson = new JSONObject(postBody);

      String dataType = reqJson.getString("DataType");
      //TODO: figure out how to set this per-adaptor
      //String cluster = reqJson.getString("Cluster");
      String adaptorClass = reqJson.getString("AdaptorClass");

      String adaptorParams = fetchOptionalString(reqJson, "AdaptorParams");
      long offset = fetchOptionalLong(reqJson, "Offset", 0);

      addCommand.append(adaptorClass).append(' ');
      addCommand.append(dataType);
      if (adaptorParams != null)
        addCommand.append(' ').append(adaptorParams);
      addCommand.append(' ').append(offset);

    } catch (JSONException e) {
      return badRequestResponse("Invalid JSON passed: '" + postBody + "', error: " + e.getMessage());
    }

    // add the adaptor
    try {
      String adaptorId = agent.processAddCommandE(addCommand.toString());

      return doGetAdaptor(agent, adaptorId, viewType);
    } catch (AdaptorException e) {
      return badRequestResponse("Could not add adaptor for postBody: '" + postBody +
              "', error: " + e.getMessage());
View Full Code Here

  @DELETE
  @Path("/{adaptorId}")
  @Produces({"text/plain"})
  public Response removeAdaptor(@Context ServletContext context,
                                @PathParam("adaptorId") String adaptorId) {
    ChukwaAgent agent = (ChukwaAgent)context.getAttribute("ChukwaAgent");

    // validate that we have an adaptorId
    if (adaptorId == null) {
      return badRequestResponse("Missing adaptorId.");
    }

    // validate that we have a valid adaptorId
    if (agent.getAdaptor(adaptorId) == null) {
      return badRequestResponse("Invalid adaptorId: " + adaptorId);
    }

    // stop the agent
    agent.stopAdaptor(adaptorId, true);
    return Response.ok().build();
  }
View Full Code Here

   */
  @GET
  @Produces({"text/xml", "text/plain"})
  public Response getAdaptors(@Context ServletContext context,
                              @QueryParam("viewType") String viewType) {
    ChukwaAgent agent = (ChukwaAgent)context.getAttribute("ChukwaAgent");
    return doGetAdaptor(agent, null, viewType);
  }
View Full Code Here

  @Path("/{adaptorId}")
  @Produces({"text/xml","text/plain"})
  public Response getAdaptor(@Context ServletContext context,
                             @QueryParam("viewType") String viewType,
                             @PathParam("adaptorId") String adaptorId) {
    ChukwaAgent agent = (ChukwaAgent)context.getAttribute("ChukwaAgent");
    return doGetAdaptor(agent, adaptorId, viewType);
  }
View Full Code Here

    // Start the collector
    System.out.println("Starting collector");
    CollectorStub.main(new String[0]);

    // Start the agent
    ChukwaAgent agent = ChukwaAgent.getAgent();

    int portno = 9093; // Default
    ChukwaAgentController cli = new ChukwaAgentController("localhost", portno);
    // ADD
    // org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.
    // CharFileTailingAdaptorUTF8NewLineEscaped
    // SysLog
    // 0 /var/log/messages
    // 0
    System.out.println("Adding adaptor");
    String adaptor = cli.add(
            "org.apache.hadoop.chukwa.datacollection.adaptor.filetailer.CharFileTailingAdaptorUTF8NewLineEscaped",
            "AutomatedTestType", "0 " + inputFile, 0);

    cli.remove(adaptor);
    System.out.println("Adaptor removed");
    agent.shutdown();
    System.out.println("Shutting down agent");
    CollectorStub.jettyServer.stop();
    System.out.println("Shutting down collector");
    Thread.sleep(2000);
  }
View Full Code Here

 
  public FileAdaptorTailer() {
   
    if (conf == null) {
      ChukwaAgent agent = ChukwaAgent.getAgent();
      if (agent != null) {
        conf = agent.getConfiguration();
        if (conf != null) {
          SAMPLE_PERIOD_MS = conf.getInt(
              "chukwaAgent.adaptor.context.switch.time",
              DEFAULT_SAMPLE_PERIOD_MS);
        }
View Full Code Here

  // consoleConnector = new ConsoleOutConnector(agent);

  protected void setUp() throws ChukwaAgent.AlreadyRunningException {
    config = new Configuration();
    agent = new ChukwaAgent(config);
    c = new ChukwaAgentController();
    connector = new ChunkCatcherConnector();
    connector.start();
  }
View Full Code Here

    ServletCollector collector2 = new ServletCollector(conf);
    Server collector1_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+1, collector1);
    Server collector2_s = TestDelayedAcks.startCollectorOnPort(conf, PORTNO+2, collector2);
    Thread.sleep(2000); //for collectors to start
   
    ChukwaAgent agent = new ChukwaAgent(conf);
    HttpConnector conn = new HttpConnector(agent);
    RetryListOfCollectors clist = new RetryListOfCollectors(conf);
    clist.add("http://localhost:"+(PORTNO+1)+"/");
    clist.add("http://localhost:"+(PORTNO+2)+"/");
    conn.setCollectors(clist);
    conn.start();
    //FIXME: somehow need to clue in commit checker which paths to check.
    //       Somehow need

    String resp = agent.processAddCommand("add adaptor_constSend = " + ConstRateAdaptor.class.getCanonicalName() +
        " testData "+ TestDelayedAcks.SEND_RATE + " 12345 0");
    assertTrue("adaptor_constSend".equals(resp));
    Thread.sleep(10 * 1000);
    collector1_s.stop();
    Thread.sleep(10 * 1000);
    SeqFileWriter.ENABLE_ROTATION_ON_CLOSE = true;

    String[] stat = agent.getAdaptorList().get("adaptor_constSend").split(" ");
    long bytesCommitted = Long.valueOf(stat[stat.length -1]);
    assertTrue(bytesCommitted > 0);
    agent.shutdown();
    conn.shutdown();
    Thread.sleep(2000); //for collectors to shut down
    collector2_s.stop();
    Thread.sleep(2000); //for collectors to shut down
   
View Full Code Here

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.