Examples of org.honu.thrift.TChunk


  public boolean isFinished() {
    return this.isDone;
  }
  private void produceChunk() throws InterruptedException {
    TChunk chunk = new TChunk();
    chunk.setSeqId(System.currentTimeMillis());
    chunk.setSource(source);
    chunk.setStreamName(streamname);
    chunk.setApplication(application);
    chunk.setDataType(dataType);
    chunk.setTags(tags);
    List<String> logEvents = new LinkedList<String>();
    chunk.setLogEvents(logEvents);


    int qSize = messages.size();
    if (qSize > maxQueueSize) {
      maxQueueSize = qSize;
    }


    int count = 0;
    // Need to be the only one to access
    // the messages list
    synchronized (wakeUpLink) {
      do {
        logEvents.add(messages.remove(0));
        count++;
      } while (!messages.isEmpty() && count < maxMessageCountPerChunk);
    }


    try {
      (new Tracer("honu.client.messageQueueSize [messages, not msec]", messages.size())).logTracer(); 
      (new Tracer("honu.client." + chunk.getApplication()  + ".messageQueueSize [messages, not msec]", messages.size())).logTracer(); 
    }catch(Exception ignored) {
      
    }
    // Drop on the floor
    // TODO instead of dropping on the floor could write to
    // a backup file
    if (chunkQueue.size() >= maxChunkQueueSize) {
      try {
        Counter.increment("honu.client.lostChunks");
        Counter.increment("honu.client.lostMessages", chunk
            .getLogEventsSize());


        Counter.increment("honu.client." + chunk.getApplication()
            + ".lostChunks");
        Counter.increment("honu.client." + chunk.getApplication()
            + ".lostMessages", chunk.getLogEventsSize());


      } catch (Exception ignored) {


      }


      kv.startMessage("HonuLostStats");
      kv.addKeyValue("lostChunk", 1);
      kv.addKeyValue("lostLines", chunk.getLogEventsSize());
      kv.addKeyValue("RecordType", chunk.getDataType());
      kv.addKeyValue("SeqId", chunk.getSeqId());
      kv.addKeyValue("period", statFrequency);
      log.error(kv.generateMessage());


      MessageManager.getInstance().updateLostDataStats(chunk);
    } else {

    return (!running && (System.currentTimeMillis() > SHUTDOWN_TIME));
  }




  public void run() {
    TChunk chunk = null;


    long timeOut = DEFAULT_POLL_TIMEOUT;
    long curr = 0l;


    while (running || (chunkQueue.size() != 0)) {
      try {
        curr = System.currentTimeMillis();


        if (shutDownNow()) {
          logApp.info("[==HONU==] Honu message sender [" + Thread.currentThread().getId()+ "] ShutdownNow");
          break;
        }


        if (curr >= nextStatPeriod ) {
          kv.startMessage("HonuSenderStats");
          kv.addKeyValue("chunkCount", chunkCount);
          kv.addKeyValue("lineCount", lineCount);
          kv.addKeyValue("exceptionCount", exceptionCount);
          kv.addKeyValue("period", statFrequency);
          log.info(kv.generateMessage());


          // Keep System.out for debug purpose
          if (log.isDebugEnabled()) {
            System.out.println(Thread.currentThread().getId() + " - "
                + new java.util.Date() + " - Chunk sent:" + chunkCount
                + " - lines:" + lineCount + " - in: 1 min+" + (curr - nextStatPeriod)
                + " ms");  
          }


          lineCount = 0;
          chunkCount = 0;
          exceptionCount = 0;
          nextStatPeriod = System.currentTimeMillis() + statFrequency;
        }


        chunk = chunkQueue.poll(timeOut, TimeUnit.MILLISECONDS);


        // If there's nothing to work on
        // increment sleep time up to maxPollTimeOut
        if (chunk == null) {
          if (timeOut < MAX_POLL_TIMEOUT) {
            timeOut += DEFAULT_POLL_TIMEOUT;
          }
          continue;
        }


        timeOut = DEFAULT_POLL_TIMEOUT;
        Tracer t = Tracer.startNewTracer("honu.client.sendChunk"); 
        try {
          (new Tracer("honu.client.chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer(); 
          (new Tracer("honu.client." + chunk.getApplication()  + ".chunkQueueSize [chunks, not msec]", chunkQueue.size())).logTracer(); 
        }catch(Exception ignored) {


        }
        
        if (System.currentTimeMillis() > leaseTs) {
          logApp.info("Time for lease renewal");
          closeConnection();
        }
        
        sendChunk(chunk);
        if (t!= null) {
          t.stopAndLogTracer(); 
        }
        chunkCount++;
        lineCount += chunk.getLogEventsSize();


        Thread.yield();
        
      }  catch (Throwable e) { 
        logApp.warn("Error in main loop",e );

Examples of org.honu.thrift.TChunk

Related Classes of org.honu.thrift.TChunk