Examples of org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor

Package org.apache.hadoop.chukwa.datacollection.adaptor

Examples of org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor

org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor
An adaptor is a component that runs within the Local Agent, producing chunks of monitoring data. An adaptor can, but need not, have an associated thread. If an adaptor lacks a thread, it needs to arrange some mechanism to periodically get control and send reports such as a callback somewhere. Adaptors must be able to stop and resume without losing data, using a byte offset in the stream. If an adaptor crashes at byte offset n, and is restarted at byte offset k, with k < n, it is allowed to send different values for bytes k through n the second time around. However, the stream must still be parseable, assuming that bytes 0-k come from the first run,and bytes k - n come from the second. Note that Adaptor implements neither equals() nor hashCode(). It is never safe to compare two adaptors with equals(). It is safe to use adaptors as hash table keys, though two distinct Adaptors will appear as two distinct keys. This is the desired behavior, since it means that messages intended for one Adaptor will never be received by another, even across Adaptor restarts.


    public void run() {
      long now = System.currentTimeMillis();


      for(String adaptorId : getAdaptorList().keySet()) {
        Adaptor adaptor = getAdaptor(adaptorId);
        if(adaptor == null) continue;


        Offset offset = adaptorPositions.get(adaptor);
        if(offset == null) continue;

View Full Code Here

      String dataType = m.group(3);
      String params = m.group(4);
      if (params == null)
        params = "";
      
      Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorClassName);
      if (adaptor == null) {
        log.warn("Error creating adaptor of class " + adaptorClassName);
        throw new AdaptorException("Can't load class " + adaptorClassName);
      }
      String coreParams = adaptor.parseArgs(dataType,params,this);
      if(coreParams == null) {
        log.warn("invalid params for adaptor: " + params);       
        throw new AdaptorException("invalid params for adaptor: " + params);
      }
      
      if(adaptorID == null) { //user didn't specify, so synthesize
        try {
         adaptorID = AdaptorNamingUtils.synthesizeAdaptorID(adaptorClassName, dataType, coreParams);
        } catch(NoSuchAlgorithmException e) {
          log.fatal("MD5 apparently doesn't work on your machine; bailing", e);
          shutdown(true);
        }
      } else if(!adaptorID.startsWith("adaptor_"))
        adaptorID = "adaptor_"+adaptorID;
      
      synchronized (adaptorsByName) {
        
        if(adaptorsByName.containsKey(adaptorID))
          return adaptorID;
        adaptorsByName.put(adaptorID, adaptor);
        adaptorPositions.put(adaptor, new Offset(offset, adaptorID));
        needNewCheckpoint = true;
        try {
          adaptor.start(adaptorID, dataType, offset, DataFactory
              .getInstance().getEventQueue());
          log.info("started a new adaptor, id = " + adaptorID + " function=["+adaptor.toString()+"]");
          ChukwaAgent.agentMetrics.adaptorCount.set(adaptorsByName.size());
          ChukwaAgent.agentMetrics.addedAdaptor.inc();
          return adaptorID;


        } catch (Exception e) {
          Adaptor failed = adaptorsByName.remove(adaptorID);
          adaptorPositions.remove(failed);
          log.warn("failed to start adaptor", e);
          if(e instanceof AdaptorException)
            throw (AdaptorException)e;
        }

View Full Code Here

   * @param number the adaptor to stop
   * @param gracefully if true, shutdown, if false, hardStop
   * @return the number of bytes synched at stop. -1 on error
   */
  public long stopAdaptor(String name, AdaptorShutdownPolicy shutdownMode) {
    Adaptor toStop;
    long offset = -1;


    // at most one thread can get past this critical section with toStop != null
    // so if multiple callers try to stop the same adaptor, all but one will
    // fail
    synchronized (adaptorsByName) {
      toStop = adaptorsByName.remove(name);
    }
    if (toStop == null) {
      log.warn("trying to stop " + name + " that isn't running");
      return offset;
    } else {
      adaptorPositions.remove(toStop);
    }
    ChukwaAgent.agentMetrics.adaptorCount.set(adaptorsByName.size());
    ChukwaAgent.agentMetrics.removedAdaptor.inc();
    
    try {
      offset = toStop.shutdown(shutdownMode);
      log.info("shutdown ["+ shutdownMode + "] on " + name + ", "
          + toStop.getCurrentStatus());
    } catch (AdaptorException e) {
      log.error("adaptor failed to stop cleanly", e);
    } finally {
      needNewCheckpoint = true;
    }

View Full Code Here

      String dataType = m.group(2);
      String params = m.group(3);
      if (params == null)
        params = "";


      Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorName);
      if (adaptor == null) {
        log.warn("Error creating adaptor from adaptor name " + adaptorName);
        return -1L;
      }


      long adaptorID;
      synchronized (adaptorsByNumber) {
        for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet()) {
          if (params.indexOf(a.getValue().getStreamName())!=-1) {
            log.warn(params + " already exist, skipping.");
            return -1;
          }
        }
        adaptorID = ++lastAdaptorNumber;
        adaptorsByNumber.put(adaptorID, adaptor);
        adaptorPositions.put(adaptor, new Offset(offset, adaptorID));
        needNewCheckpoint = true;
        try {
          adaptor.start(adaptorID, dataType, params, offset, DataFactory
              .getInstance().getEventQueue());
          log.info("started a new adaptor, id = " + adaptorID);
          ChukwaAgent.agentMetrics.adaptorCount.set(adaptorsByNumber.size());
          ChukwaAgent.agentMetrics.addedAdaptor.inc();
          return adaptorID;

View Full Code Here

      PrintWriter out = new PrintWriter(new BufferedWriter(
          new OutputStreamWriter(fos)));


      for (Map.Entry<Adaptor, Offset> stat : adaptorPositions.entrySet()) {
        try {
          Adaptor a = stat.getKey();
          out.print("ADD " + a.getClass().getCanonicalName());
          out.println(" " + a.getCurrentStatus());
        } catch (AdaptorException e) {
          e.printStackTrace();
        }// don't try to recover from bad adaptor yet
      }

View Full Code Here

   * @param number the adaptor to stop
   * @param gracefully if true, shutdown, if false, hardStop
   * @return the number of bytes synched at stop. -1 on error
   */
  public long stopAdaptor(long number, boolean gracefully) {
    Adaptor toStop;
    long offset = -1;


    // at most one thread can get past this critical section with toStop != null
    // so if multiple callers try to stop the same adaptor, all but one will
    // fail
    synchronized (adaptorsByNumber) {
      toStop = adaptorsByNumber.remove(number);
    }
    if (toStop == null) {
      log.warn("trying to stop adaptor " + number + " that isn't running");
      return offset;
    } else {
      adaptorPositions.remove(toStop);
    }
    ChukwaAgent.agentMetrics.adaptorCount.set(adaptorsByNumber.size());
    ChukwaAgent.agentMetrics.removedAdaptor.inc();
    
    try {
      if (gracefully) {
        offset = toStop.shutdown();
        log.info("shutdown on adaptor: " + number + ", "
            + toStop.getCurrentStatus());
      } else {
        toStop.hardStop();
        log.info("hardStop on adaptorId: " + number + ", "
            + toStop.getCurrentStatus());
      }
    } catch (AdaptorException e) {
      log.error("adaptor failed to stop cleanly", e);
    } finally {
      needNewCheckpoint = true;

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.chukwa.datacollection.adaptor.Adaptor

org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent

org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent$StatsCollectorTask

org.apache.hadoop.chukwa.datacollection.agent.rest.AdaptorController

org.apache.hadoop.chukwa.datacollection.agent.TestCmd

org.apache.hadoop.chukwa.datacollection.sender.TestAcksOnFailure

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.