Package org.honu.util

Examples of org.honu.util.Tracer


  /**
   *  Best effort, there's no guarantee that chunks
   *  have really been written to disk
   */
  public CommitStatus add(List<Chunk> chunks) throws WriterException {
    Tracer t = Tracer.startNewTracer("honu.server." + group +".addToList");
    long now = System.currentTimeMillis();
    if (chunks != null) {
      try {
        chunksWrittenThisRotate = true;
        ChukwaArchiveKey archiveKey = new ChukwaArchiveKey();

        for (Chunk chunk : chunks) {
          archiveKey.setTimePartition(timePeriod);
          archiveKey.setDataType(chunk.getDataType());
          archiveKey.setStreamName(chunk.getTags() + "/" + chunk.getSource()
              + "/" + chunk.getStreamName());
          archiveKey.setSeqId(chunk.getSeqID());

          if (chunk != null) {
            seqFileWriter.append(archiveKey, chunk);
            // compute size for stats
            dataSize += chunk.getData().length;
          }
        }
       
        long end = System.currentTimeMillis();
        if (log.isDebugEnabled()) {
          log.debug(group+"- duration=" + (end-now) + " size=" + chunks.size());
        }

      } catch (IOException e) {
        if (t!= null) {
          t.stopAndLogTracer();
        }
       
        writeChunkRetries--;
        log.error(group +"- Could not save the chunk. ", e);

        if (writeChunkRetries < 0) {
          log
          .fatal(group +"- Too many IOException when trying to write a chunk, Collector is going to exit!");
          DaemonWatcher.bailout(-1);
        }
        throw new WriterException(e);
      }
    }
 
    if (t!= null) {
      t.stopAndLogTracer();
    }
    
    return COMMIT_OK;
  }
View Full Code Here


    
    return COMMIT_OK;
  }

  protected void rotate() {
    Tracer t = Tracer.startNewTracer("honu.server." + group +".rotateDataSink");
    isRunning = true;
    calendar.setTimeInMillis(System.currentTimeMillis());
    log.info(group +"- start Date [" + calendar.getTime() + "]");
    log.info(group +"- Rotate from " + Thread.currentThread().getName());

    String newName = day.format(calendar.getTime());
    newName += localHostAddr + new java.rmi.server.UID().toString();
    newName = newName.replace("-", "");
    newName = newName.replace(":", "");
    // newName = newName.replace(".", "");
    newName = localOutputDir + "/" + newName.trim();
   
    try {
      FSDataOutputStream previousOutputStr = currentOutputStr;
      Path previousPath = currentPath;
      String previousFileName = currentFileName;

      if (previousOutputStr != null) {
        seqFileWriter.close();
        previousOutputStr.close();
        if (chunksWrittenThisRotate) {
          fs.rename(previousPath, new Path(previousFileName + ".done"));
          fileQueue.add(previousFileName + ".done");
        } else {
          log.info(group+"- no chunks written to " + previousPath + ", deleting");
          fs.delete(previousPath, false);
        }
      }
     
      Path newOutputPath = new Path(newName + ".chukwa");
      FSDataOutputStream newOutputStr = fs.create(newOutputPath);

      currentOutputStr = newOutputStr;
      currentPath = newOutputPath;
      currentFileName = newName;
      chunksWrittenThisRotate = false;
     
      if (codec != null) {
        seqFileWriter = SequenceFile.createWriter(conf, newOutputStr,
            ChukwaArchiveKey.class, ChunkImpl.class,
            SequenceFile.CompressionType.BLOCK, codec);
      } else {
        seqFileWriter = SequenceFile.createWriter(conf, newOutputStr,
            ChukwaArchiveKey.class, ChunkImpl.class,
            SequenceFile.CompressionType.NONE, codec);
      }
     
    } catch (Throwable e) {
      if (t!= null) {
        t.stopAndLogTracer();
      }
     
      log.fatal(group+"- Throwable Exception in rotate. Exiting!", e);
      // Shutting down the collector
      // Watchdog will re-start it automatically
      DaemonWatcher.bailout(-1);
    }

    // Check for disk space
    File directory4Space = new File(localOutputDir);
    long totalSpace = directory4Space.getTotalSpace();
    long freeSpace = directory4Space.getFreeSpace();
    long minFreeAvailable = (totalSpace * minPercentFreeDisk) /100;

    if (log.isDebugEnabled()) {
      log.debug( group +"- Directory: " + localOutputDir + ", totalSpace: " + totalSpace
          + ", freeSpace: " + freeSpace + ", minFreeAvailable: " + minFreeAvailable
          + ", percentFreeDisk: " + minPercentFreeDisk);
    }

    if (freeSpace < minFreeAvailable) {
      log.fatal( group +"- No space left on device, Bail out!");
      DaemonWatcher.bailout(-1);
    }
    nextRotate = System.currentTimeMillis() + rotateInterval;
   
    if (t!= null) {
      t.stopAndLogTracer();
    }
  }
View Full Code Here

      result.setMessage(""+tChunk.getSeqId());
      result.setResultCode(ResultCode.OK);
      return result;
    }
   
     Tracer t = Tracer.startNewTracer("honu.server.processChunk");
    //this.counters.get(chunkCountField).incrementAndGet();
   
    ChunkBuilder cb = new ChunkBuilder();
    List<String> logEvents = tChunk.getLogEvents();
   
   
    for(String logEvent :logEvents) {
      cb.addRecord(logEvent.getBytes());
    }
   
    Chunk c = cb.getChunk();
    c.setApplication(tChunk.getApplication());
    c.setDataType(tChunk.getDataType());
    c.setSeqID(tChunk.getSeqId());
    c.setSource(tChunk.getSource());
    c.setTags(tChunk.getTags());
   
    if (isDebug) {
      System.out.println("\n\t ===============");
      System.out.println("tChunk.getApplication() :" + tChunk.getApplication());
      System.out.println("tChunk.getDataType() :" + tChunk.getDataType());
      System.out.println("tChunk.getSeqId() :" + tChunk.getSeqId());
      System.out.println("tChunk.getSource() :" + tChunk.getSource());
      System.out.println("tChunk.getStreamName() :" + tChunk.getStreamName());
      System.out.println("tChunk.getTags() :" + tChunk.getTags());
     
     
      System.out.println("c.getApplication() :" + c.getApplication());
      System.out.println("c.getDataType() :" + c.getDataType());
      System.out.println("c.getSeqID() :" + c.getSeqID());
      System.out.println("c.getSource() :" + c.getSource());
      System.out.println("c.getTags() :" + c.getTags());
      System.out.println("c.getData()" + new String( c.getData()));
     
    }
 
   
   
    boolean addResult = false;
   
    try {
        addResult = chunkQueue.offer(c, 2000, TimeUnit.MILLISECONDS);
    }catch (OutOfMemoryError ex) {
      ex.printStackTrace();
      DaemonWatcher.bailout(-1);
    }
    catch (Throwable e) {
      e.printStackTrace();
      addResult = false;
    }
 
    Result result = new Result();
   
    if (addResult) {
      try {
        Counter.increment("honu.server.chunkCount");
        Counter.increment("honu.server.logCount",logEvents.size());
       
        Counter.increment("honu.server."+ tChunk.getApplication() +".chunkCount");
        Counter.increment("honu.server."+ tChunk.getApplication() +".logCount",logEvents.size())
      
        (new Tracer("honu.server.chunkSize [messages, not msec]", logEvents.size())).logTracer();
        (new Tracer("honu.server." + tChunk.getApplication()  + ".chunkSize [messages, not msec]", logEvents.size())).logTracer();
      }catch(Exception ignored) {
       
      }
     
   
      result.setMessage(""+tChunk.getSeqId());
      result.setResultCode(ResultCode.OK);
    } else {
      try {
        Counter.increment("honu.server.tryLater");
        Counter.increment("honu.server."+ tChunk.getApplication() +".tryLater");  
      }catch(Exception ignored) {
       
      }
     
      result.setMessage(""+tChunk.getSeqId());
      result.setResultCode(ResultCode.TRY_LATER);
    }
    if (t!= null) {
      t.stopAndLogTracer();
    }
   
    return result;
  }
View Full Code Here

        count++;
      } while (!messages.isEmpty() && count < maxMessageCountPerChunk);
    }

    try {
      (new Tracer("honu.client.messageQueueSize [messages, not msec]", messages.size())).logTracer();
      (new Tracer("honu.client." + chunk.getApplication()  + ".messageQueueSize [messages, not msec]", messages.size())).logTracer();
    }catch(Exception ignored) {
     
    }
    // Drop on the floor
    // TODO instead of dropping on the floor could write to
View Full Code Here

          }
          continue;
        }

        timeOut = DEFAULT_POLL_TIMEOUT;
        Tracer t = Tracer.startNewTracer("honu.client.sendChunk");
        try {
          (new Tracer("honu.client.chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer();
          (new Tracer("honu.client." + chunk.getApplication()  + ".chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer();
        }catch(Exception ignored) {

        }
       
        if (System.currentTimeMillis() > leaseTs) {
          logApp.info("Time for lease renewal");
          closeConnection();
        }
       
        sendChunk(chunk);
        if (t!= null) {
          t.stopAndLogTracer();
        }
        chunkCount++;
        lineCount += chunk.getLogEventsSize();

        Thread.yield();
View Full Code Here

    Result result = null;


    do {
      try {
        Tracer t = Tracer.startNewTracer("honu.client.processChunk");
        try {

          if ((collector == null) || (tryLaterErrorCount >= 3)) {
            tryLaterErrorCount = 0;
            initConnection();
          }

          result = collector.process(chunk);
          if (t!= null){
            t.stopAndLogTracer();
          }
        

          if (result.getResultCode() == ResultCode.TRY_LATER) {
            Counter.increment("honu.client.tryLater");
            Counter.increment("honu.client."+ chunk.getApplication() +".tryLater");

            tryLaterErrorCount++;

            // If there's more than 3 TRY_LATER
            // on a collector the switch
            if (tryLaterErrorCount > 3) {
              collector = null;
              errorCount++;
            } else {
              randomSleep(5 * tryLaterErrorCount, true);
            }

          } else {
            Counter.increment("honu.client.chunkCount");
            Counter.increment("honu.client.logCount",chunk.getLogEventsSize());

            Counter.increment("honu.client."+ chunk.getApplication() +".chunkCount");
            Counter.increment("honu.client."+ chunk.getApplication() +".logCount",chunk.getLogEventsSize());
          }
        } catch (Throwable e) {

          try {
            log.warn("exception in sendChunk", e);
            if (t != null) {
              t.stopAndLogTracer();
            }

            Counter.increment("honu.client.exceptionCount");

            exceptionCount++;
View Full Code Here

TOP

Related Classes of org.honu.util.Tracer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.