Examples of org.apache.flume.Transaction

org.apache.flume.Transaction
ut(event) or ch.take() ... tx.commit(); } catch (ChannelException ex) { tx.rollback(); ... } finally { tx.close(); }
Depending upon the implementation of the channel, the transaction semantics may be strong, or best-effort only.
@see org.apache.flume.Channel


  public static Set<String> takeEvents(Channel channel,
          int batchSize, int numEvents) throws Exception {
    Set<String> result = Sets.newHashSet();
    for (int i = 0; i < numEvents; i += batchSize) {
      Transaction transaction = channel.getTransaction();
      try {
        transaction.begin();
        for (int j = 0; j < batchSize; j++) {
          Event event = null;
          try {
            event = channel.take();
          } catch (ChannelException ex) {
            Assert.assertTrue(ex.getMessage().startsWith(
                "Take list for FileBackedTransaction, capacity"));
            transaction.commit();
            return result;
          }
          if (event == null) {
            transaction.commit();
            return result;
          }
          result.add(new String(event.getBody(), Charsets.UTF_8));
        }
        transaction.commit();
      } catch (Exception ex) {
        transaction.rollback();
        throw ex;
      } finally {
        transaction.close();
      }


    }
    return result;
  }

View Full Code Here

  public static Set<String> putEvents(Channel channel, String prefix,
          int batchSize, int numEvents, boolean untilCapacityIsReached)
              throws Exception {
    Set<String> result = Sets.newHashSet();
    for (int i = 0; i < numEvents; i += batchSize) {
      Transaction transaction = channel.getTransaction();
      transaction.begin();
      try {
        Set<String> batch = Sets.newHashSet();
        for (int j = 0; j < batchSize; j++) {
          String s = prefix + "-" + i + "-" + j + "-" + UUID.randomUUID();
          Event event = EventBuilder.withBody(s.getBytes(Charsets.UTF_8));
          channel.put(event);
          batch.add(s);
        }
        transaction.commit();
        result.addAll(batch);
      } catch (Exception ex) {
        transaction.rollback();
        if(untilCapacityIsReached && ex instanceof ChannelException &&
            ("The channel has reached it's capacity. " 
                + "This might be the result of a sink on the channel having too "
                + "low of batch size, a downstream system running slower than "
                + "normal, or that the channel capacity is just too low. "
                + "[channel=" +channel.getName() + "]").
              equals(ex.getMessage())) {
          break;
        }
        throw ex;
      } finally {
        transaction.close();
      }
    }
    return result;
  }

View Full Code Here


  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction txn = channel.getTransaction();
    List<Row> actions = new LinkedList<Row>();
    List<Increment> incs = new LinkedList<Increment>();
    txn.begin();
    for(long i = 0; i < batchSize; i++) {
      Event event = channel.take();
      if(event == null){
        status = Status.BACKOFF;
        counterGroup.incrementAndGet("channel.underflow");

View Full Code Here

    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(
        Bytes.toBytes(valBase));
    channel.put(e);
    tx.commit();
    tx.close();


    sink.process();
    sink.stop();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);
    byte[][] results = getResults(table, 1);

View Full Code Here


  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();


    try {
      transaction.begin();


      verifyConnection();


      List<Event> batch = Lists.newLinkedList();


      for (int i = 0; i < client.getBatchSize(); i++) {
        Event event = channel.take();


        if (event == null) {
          break;
        }


        batch.add(event);
      }


      int size = batch.size();
      int batchSize = client.getBatchSize();


      if (size == 0) {
        sinkCounter.incrementBatchEmptyCount();
        status = Status.BACKOFF;
      } else {
        if (size < batchSize) {
          sinkCounter.incrementBatchUnderflowCount();
        } else {
          sinkCounter.incrementBatchCompleteCount();
        }
        sinkCounter.addToEventDrainAttemptCount(size);
        client.appendBatch(batch);
      }


      transaction.commit();
      sinkCounter.addToEventDrainSuccessCount(size);


    } catch (Throwable t) {
      transaction.rollback();
      if (t instanceof Error) {
        throw (Error) t;
      } else if (t instanceof ChannelException) {
        logger.error("Avro Sink " + getName() + ": Unable to get event from" +
            " channel " + channel.getName() + ". Exception follows.", t);
        status = Status.BACKOFF;
      } else {
        destroyConnection();
        throw new EventDeliveryException("Failed to send events", t);
      }
    } finally {
      transaction.close();
    }


    return status;
  }

View Full Code Here

    Configurables.configure(sink, ctx);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    sink.stop();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);
    byte[][] results = getResults(table, 3);
    byte[] out;

View Full Code Here

    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    int count = 0;
    Status status = Status.READY;
    while(status != Status.BACKOFF){
      count++;
      status = sink.process();

View Full Code Here

    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);
    byte[][] results = getResults(table, 2);
    byte[] out;
    int found = 0;

View Full Code Here

    ctx.put("batchSize", "100");
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    for(int i = 0; i < 3; i++){
      Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
      channel.put(e);
    }
    tx.commit();
    tx.close();
    sink.process();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);
    byte[][] results = getResults(table, 2);
    byte[] out;
    int found = 0;

View Full Code Here

  @Override
  public Status process() throws EventDeliveryException {
    logger.debug("processing...");
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction txn = channel.getTransaction();
    try {
      txn.begin();
      String indexName = getIndexName();
      BulkRequestBuilder bulkRequest = client.prepareBulk();
      for (int i = 0; i < batchSize; i++) {
        Event event = channel.take();


        if (event == null) {
          break;
        }


        XContentBuilder builder = serializer.getContentBuilder(event);
        IndexRequestBuilder request = client.prepareIndex(indexName, indexType)
            .setSource(builder);


        if (ttlMs > 0) {
          request.setTTL(ttlMs);
        }


        bulkRequest.add(request);
      }


      int size = bulkRequest.numberOfActions();
      if (size <= 0) {
        sinkCounter.incrementBatchEmptyCount();
        counterGroup.incrementAndGet("channel.underflow");
        status = Status.BACKOFF;
      } else {
        if (size < batchSize) {
          sinkCounter.incrementBatchUnderflowCount();
          status = Status.BACKOFF;
        } else {
          sinkCounter.incrementBatchCompleteCount();
        }


        sinkCounter.addToEventDrainAttemptCount(size);


        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
          throw new EventDeliveryException(bulkResponse.buildFailureMessage());
        }
      }
      txn.commit();
      sinkCounter.addToEventDrainSuccessCount(size);
      counterGroup.incrementAndGet("transaction.success");
    } catch (Throwable ex) {
      try {
        txn.rollback();
        counterGroup.incrementAndGet("transaction.rollback");
      } catch (Exception ex2) {
        logger.error(
            "Exception in rollback. Rollback might not have been successful.",
            ex2);
      }


      if (ex instanceof Error || ex instanceof RuntimeException) {
        logger.error("Failed to commit transaction. Transaction rolled back.",
            ex);
        Throwables.propagate(ex);
      } else {
        logger.error("Failed to commit transaction. Transaction rolled back.",
            ex);
        throw new EventDeliveryException(
            "Failed to commit transaction. Transaction rolled back.", ex);
      }
    } finally {
      txn.close();
    }
    return status;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Transaction

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.ChannelProcessor

org.apache.flume.channel.ChannelUtils

org.apache.flume.channel.FanoutChannel$wrapperTransaction

org.apache.flume.channel.file.TestFileChannel

org.apache.flume.channel.file.TestFileChannelRestart

org.apache.flume.channel.file.TestFileChannelRollback

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.