Package org.apache.flume

Examples of org.apache.flume.ChannelException


    @Override
    protected Event doTake() throws InterruptedException {
      channelCounter.incrementEventTakeAttemptCount();
      if(takeList.remainingCapacity() == 0) {
        throw new ChannelException("Take list for FileBackedTransaction, capacity " +
            takeList.size() + " full, consider committing more frequently, " +
            "increasing capacity, or increasing thread count. "
               + channelNameDescriptor);
      }
      if(!log.tryLockShared()) {
        throw new ChannelException("Failed to obtain lock for writing to the "
            + "log. Try increasing the log write timeout value. " +
            channelNameDescriptor);
      }

      /*
       * 1. Take an event which is in the queue.
       * 2. If getting that event does not throw NoopRecordException,
       *    then return it.
       * 3. Else try to retrieve the next event from the queue
       * 4. Repeat 2 and 3 until queue is empty or an event is returned.
       */

      try {
        while (true) {
          FlumeEventPointer ptr = queue.removeHead(transactionID);
          if (ptr == null) {
            return null;
          } else {
            try {
              // first add to takeList so that if write to disk
              // fails rollback actually does it's work
              Preconditions.checkState(takeList.offer(ptr),
                "takeList offer failed "
                  + channelNameDescriptor);
              log.take(transactionID, ptr); // write take to disk
              Event event = log.get(ptr);
              return event;
            } catch (IOException e) {
              throw new ChannelException("Take failed due to IO error "
                + channelNameDescriptor, e);
            } catch (NoopRecordException e) {
              LOG.warn("Corrupt record replaced by File Channel Integrity " +
                "tool found. Will retrieve next event", e);
              takeList.remove(ptr);
View Full Code Here


      int takes = takeList.size();
      if(puts > 0) {
        Preconditions.checkState(takes == 0, "nonzero puts and takes "
                + channelNameDescriptor);
        if(!log.tryLockShared()) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        try {
          log.commitPut(transactionID);
          channelCounter.addToEventPutSuccessCount(puts);
          synchronized (queue) {
            while(!putList.isEmpty()) {
              if(!queue.addTail(putList.removeFirst())) {
                StringBuilder msg = new StringBuilder();
                msg.append("Queue add failed, this shouldn't be able to ");
                msg.append("happen. A portion of the transaction has been ");
                msg.append("added to the queue but the remaining portion ");
                msg.append("cannot be added. Those messages will be consumed ");
                msg.append("despite this transaction failing. Please report.");
                msg.append(channelNameDescriptor);
                LOG.error(msg.toString());
                Preconditions.checkState(false, msg.toString());
              }
            }
            queue.completeTransaction(transactionID);
          }
        } catch (IOException e) {
          throw new ChannelException("Commit failed due to IO error "
                  + channelNameDescriptor, e);
        } finally {
          log.unlockShared();
        }

      } else if (takes > 0) {
        if(!log.tryLockShared()) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        try {
          log.commitTake(transactionID);
          queue.completeTransaction(transactionID);
          channelCounter.addToEventTakeSuccessCount(takes);
        } catch (IOException e) {
          throw new ChannelException("Commit failed due to IO error "
              + channelNameDescriptor, e);
        } finally {
          log.unlockShared();
        }
        queueRemaining.release(takes);
View Full Code Here

      int puts = putList.size();
      int takes = takeList.size();
      boolean lockAcquired = log.tryLockShared();
      try {
        if(!lockAcquired) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        if(takes > 0) {
          Preconditions.checkState(puts == 0, "nonzero puts and takes "
              + channelNameDescriptor);
          synchronized (queue) {
            while (!takeList.isEmpty()) {
              Preconditions.checkState(queue.addHead(takeList.removeLast()),
                  "Queue add failed, this shouldn't be able to happen "
                      + channelNameDescriptor);
            }
          }
        }
        putList.clear();
        takeList.clear();
        queue.completeTransaction(transactionID);
        channelCounter.setChannelSize(queue.getSize());
        log.rollback(transactionID);
      } catch (IOException e) {
        throw new ChannelException("Commit failed due to IO error "
            + channelNameDescriptor, e);
      } finally {
        if(lockAcquired) {
          log.unlockShared();
        }
View Full Code Here

   * @throws IOException
   */
    private synchronized void roll(int index, ByteBuffer buffer)
      throws IOException {
    if (!tryLockShared()) {
      throw new ChannelException("Failed to obtain lock for writing to the "
          + "log. Try increasing the log write timeout value. " +
          channelNameDescriptor);
    }

    try {
View Full Code Here

   */
  @Test(expected = ChannelException.class)
  public void testExceptionFromGetTransaction() {
    // create a channel which unexpectedly throws a ChEx on getTransaction()
    Channel ch = mock(Channel.class);
    when(ch.getTransaction()).thenThrow(new ChannelException("doh!"));

    ChannelSelector sel = new ReplicatingChannelSelector();
    sel.setChannels(Lists.newArrayList(ch));
    ChannelProcessor proc = new ChannelProcessor(sel);

View Full Code Here

   * @throws IOException
   */
    private synchronized void roll(int index, ByteBuffer buffer)
      throws IOException {
    if (!tryLockShared()) {
      throw new ChannelException("Failed to obtain lock for writing to the "
          + "log. Try increasing the log write timeout value. " +
          channelNameDescriptor);
    }

    try {
View Full Code Here

    }
    @Override
    protected void doPut(Event event) throws InterruptedException {
      channelCounter.incrementEventPutAttemptCount();
      if(putList.remainingCapacity() == 0) {
        throw new ChannelException("Put queue for FileBackedTransaction " +
            "of capacity " + putList.size() + " full, consider " +
            "committing more frequently, increasing capacity or " +
            "increasing thread count. " + channelNameDescriptor);
      }
      // this does not need to be in the critical section as it does not
      // modify the structure of the log or queue.
      if(!queueRemaining.tryAcquire(keepAlive, TimeUnit.SECONDS)) {
        throw new ChannelException("The channel has reached it's capacity. "
            + "This might be the result of a sink on the channel having too "
            + "low of batch size, a downstream system running slower than "
            + "normal, or that the channel capacity is just too low. "
            + channelNameDescriptor);
      }
      boolean success = false;
      boolean lockAcquired = log.tryLockShared();
      try {
        if(!lockAcquired) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        FlumeEventPointer ptr = log.put(transactionID, event);
        Preconditions.checkState(putList.offer(ptr), "putList offer failed "
             + channelNameDescriptor);
        queue.addWithoutCommit(ptr, transactionID);
        success = true;
      } catch (IOException e) {
        throw new ChannelException("Put failed due to IO error "
                + channelNameDescriptor, e);
      } finally {
        if(lockAcquired) {
          log.unlockShared();
        }
View Full Code Here

    @Override
    protected Event doTake() throws InterruptedException {
      channelCounter.incrementEventTakeAttemptCount();
      if(takeList.remainingCapacity() == 0) {
        throw new ChannelException("Take list for FileBackedTransaction, capacity " +
            takeList.size() + " full, consider committing more frequently, " +
            "increasing capacity, or increasing thread count. "
               + channelNameDescriptor);
      }
      if(!log.tryLockShared()) {
        throw new ChannelException("Failed to obtain lock for writing to the "
            + "log. Try increasing the log write timeout value. " +
            channelNameDescriptor);
      }
      try {
        FlumeEventPointer ptr = queue.removeHead(transactionID);
        if(ptr != null) {
          try {
            // first add to takeList so that if write to disk
            // fails rollback actually does it's work
            Preconditions.checkState(takeList.offer(ptr), "takeList offer failed "
                 + channelNameDescriptor);
            log.take(transactionID, ptr); // write take to disk
            Event event = log.get(ptr);
            return event;
          } catch (IOException e) {
            throw new ChannelException("Take failed due to IO error "
                    + channelNameDescriptor, e);
          }
        }
        return null;
      } finally {
View Full Code Here

      int takes = takeList.size();
      if(puts > 0) {
        Preconditions.checkState(takes == 0, "nonzero puts and takes "
                + channelNameDescriptor);
        if(!log.tryLockShared()) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        try {
          log.commitPut(transactionID);
          channelCounter.addToEventPutSuccessCount(puts);
          synchronized (queue) {
            while(!putList.isEmpty()) {
              if(!queue.addTail(putList.removeFirst())) {
                StringBuilder msg = new StringBuilder();
                msg.append("Queue add failed, this shouldn't be able to ");
                msg.append("happen. A portion of the transaction has been ");
                msg.append("added to the queue but the remaining portion ");
                msg.append("cannot be added. Those messages will be consumed ");
                msg.append("despite this transaction failing. Please report.");
                msg.append(channelNameDescriptor);
                LOG.error(msg.toString());
                Preconditions.checkState(false, msg.toString());
              }
            }
            queue.completeTransaction(transactionID);
          }
        } catch (IOException e) {
          throw new ChannelException("Commit failed due to IO error "
                  + channelNameDescriptor, e);
        } finally {
          log.unlockShared();
        }

      } else if (takes > 0) {
        if(!log.tryLockShared()) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        try {
          log.commitTake(transactionID);
          queue.completeTransaction(transactionID);
          channelCounter.addToEventTakeSuccessCount(takes);
        } catch (IOException e) {
          throw new ChannelException("Commit failed due to IO error "
              + channelNameDescriptor, e);
        } finally {
          log.unlockShared();
        }
        queueRemaining.release(takes);
View Full Code Here

      int puts = putList.size();
      int takes = takeList.size();
      boolean lockAcquired = log.tryLockShared();
      try {
        if(!lockAcquired) {
          throw new ChannelException("Failed to obtain lock for writing to the "
              + "log. Try increasing the log write timeout value. " +
              channelNameDescriptor);
        }
        if(takes > 0) {
          Preconditions.checkState(puts == 0, "nonzero puts and takes "
              + channelNameDescriptor);
          synchronized (queue) {
            while (!takeList.isEmpty()) {
              Preconditions.checkState(queue.addHead(takeList.removeLast()),
                  "Queue add failed, this shouldn't be able to happen "
                      + channelNameDescriptor);
            }
          }
        }
        putList.clear();
        takeList.clear();
        queue.completeTransaction(transactionID);
        channelCounter.setChannelSize(queue.getSize());
        log.rollback(transactionID);
      } catch (IOException e) {
        throw new ChannelException("Commit failed due to IO error "
            + channelNameDescriptor, e);
      } finally {
        if(lockAcquired) {
          log.unlockShared();
        }
View Full Code Here

TOP

Related Classes of org.apache.flume.ChannelException

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.