Examples of org.apache.flume.Transaction

org.apache.flume.Transaction
ut(event) or ch.take() ... tx.commit(); } catch (ChannelException ex) { tx.rollback(); ... } finally { tx.close(); }
Depending upon the implementation of the channel, the transaction semantics may be strong, or best-effort only.
@see org.apache.flume.Channel

      syslogSocket.getOutputStream().write(bodyWithTandH.getBytes());
      syslogSocket.close();
    }


    List<Event> channelEvents = new ArrayList<Event>();
    Transaction txn = channel.getTransaction();
    txn.begin();
    for (int i = 0; i < 10; i++) {
      Event e = channel.take();
      if (e == null) {
        throw new NullPointerException("Event is null");
      }
      channelEvents.add(e);
    }


    try {
      txn.commit();
    } catch (Throwable t) {
      txn.rollback();
    } finally {
      txn.close();
    }


    source.stop();
    for (Event e : channelEvents) {
      Assert.assertNotNull(e);

View Full Code Here

  @Override
  public Status process() throws EventDeliveryException {
    logger.debug("processing...");
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction txn = channel.getTransaction();
    try {
      txn.begin();
      int count;
      for (count = 0; count < batchSize; ++count) {
        Event event = channel.take();


        if (event == null) {
          break;
        }
        String realIndexType = BucketPath.escapeString(indexType, event.getHeaders());
        client.addEvent(event, indexNameBuilder, realIndexType, ttlMs);
      }


      if (count <= 0) {
        sinkCounter.incrementBatchEmptyCount();
        counterGroup.incrementAndGet("channel.underflow");
        status = Status.BACKOFF;
      } else {
        if (count < batchSize) {
          sinkCounter.incrementBatchUnderflowCount();
          status = Status.BACKOFF;
        } else {
          sinkCounter.incrementBatchCompleteCount();
        }


        sinkCounter.addToEventDrainAttemptCount(count);
        client.execute();
      }
      txn.commit();
      sinkCounter.addToEventDrainSuccessCount(count);
      counterGroup.incrementAndGet("transaction.success");
    } catch (Throwable ex) {
      try {
        txn.rollback();
        counterGroup.incrementAndGet("transaction.rollback");
      } catch (Exception ex2) {
        logger.error(
            "Exception in rollback. Rollback might not have been successful.",
            ex2);
      }


      if (ex instanceof Error || ex instanceof RuntimeException) {
        logger.error("Failed to commit transaction. Transaction rolled back.",
            ex);
        Throwables.propagate(ex);
      } else {
        logger.error("Failed to commit transaction. Transaction rolled back.",
            ex);
        throw new EventDeliveryException(
            "Failed to commit transaction. Transaction rolled back.", ex);
      }
    } finally {
      txn.close();
    }
    return status;
  }

View Full Code Here

  @Test
  public void shouldIndexOneEvent() throws Exception {
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);


    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody("event #1 or 1".getBytes());
    channel.put(event);
    tx.commit();
    tx.close();


    fixture.process();
    fixture.stop();
    client.admin().indices()
        .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

View Full Code Here

  @Test
  public void shouldIndexComplexJsonEvent() throws Exception {
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);


    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody("{\"event\":\"json content\"}".getBytes());
    channel.put(event);
    tx.commit();
    tx.close();


    fixture.process();
    fixture.stop();
    client.admin().indices()
            .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

View Full Code Here

    Channel channel = bindAndStartChannel(fixture);


    int numberOfEvents = 5;
    Event[] events = new Event[numberOfEvents];


    Transaction tx = channel.getTransaction();
    tx.begin();
    for (int i = 0; i < numberOfEvents; i++) {
      String body = "event #" + i + " of " + numberOfEvents;
      Event event = EventBuilder.withBody(body.getBytes());
      events[i] = event;
      channel.put(event);
    }
    tx.commit();
    tx.close();


    fixture.process();
    fixture.stop();
    client.admin().indices()
        .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

View Full Code Here

    Channel channel = bindAndStartChannel(fixture);


    int numberOfEvents = 5;
    Event[] events = new Event[numberOfEvents];


    Transaction tx = channel.getTransaction();
    tx.begin();
    for (int i = 0; i < numberOfEvents; i++) {
      String body = "event #" + i + " of " + numberOfEvents;
      Event event = EventBuilder.withBody(body.getBytes());
      events[i] = event;
      channel.put(event);
    }
    tx.commit();
    tx.close();


    int count = 0;
    Status status = Status.READY;
    while (status != Status.BACKOFF) {
      count++;

View Full Code Here

        CustomElasticSearchIndexRequestBuilderFactory.class.getName());


    fixture.configure(new Context(parameters));


    Channel channel = bindAndStartChannel(fixture);
    Transaction tx = channel.getTransaction();
    tx.begin();
    String body = "{ foo: \"bar\" }";
    Event event = EventBuilder.withBody(body.getBytes());
    channel.put(event);
    tx.commit();
    tx.close();


    fixture.process();
    fixture.stop();


    assertEquals(fixture.getIndexName() + "-05_17_36_789",

View Full Code Here


  @Override
  public Status process() throws EventDeliveryException {
    Status status = Status.READY;
    Channel channel = getChannel();
    Transaction transaction = channel.getTransaction();


    if(resetConnectionFlag.get()) {
      resetConnection();
      // if the time to reset is long and the timeout is short
      // this may cancel the next reset request
      // this should however not be an issue
      resetConnectionFlag.set(false);
    }


    try {
      transaction.begin();


      verifyConnection();


      List<Event> batch = Lists.newLinkedList();


      for (int i = 0; i < client.getBatchSize(); i++) {
        Event event = channel.take();


        if (event == null) {
          break;
        }


        batch.add(event);
      }


      int size = batch.size();
      int batchSize = client.getBatchSize();


      if (size == 0) {
        sinkCounter.incrementBatchEmptyCount();
        status = Status.BACKOFF;
      } else {
        if (size < batchSize) {
          sinkCounter.incrementBatchUnderflowCount();
        } else {
          sinkCounter.incrementBatchCompleteCount();
        }
        sinkCounter.addToEventDrainAttemptCount(size);
        client.appendBatch(batch);
      }


      transaction.commit();
      sinkCounter.addToEventDrainSuccessCount(size);


    } catch (Throwable t) {
      transaction.rollback();
      if (t instanceof Error) {
        throw (Error) t;
      } else if (t instanceof ChannelException) {
        logger.error("Rpc Sink " + getName() + ": Unable to get event from" +
            " channel " + channel.getName() + ". Exception follows.", t);
        status = Status.BACKOFF;
      } else {
        destroyConnection();
        throw new EventDeliveryException("Failed to send events", t);
      }
    } finally {
      transaction.close();
    }


    return status;
  }

View Full Code Here

    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel to roll twice
    for (i = 1; i <= (rollCount*10)/batchSize; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();


      // execute sink to process the events
      sink.process();
    }

View Full Code Here

    Calendar eventDate = Calendar.getInstance();
    List<String> bodies = Lists.newArrayList();


    // push the event batches into channel
    for (i = 1; i < 4; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2011, i, i, i, 0); // yy mm dd
        event.getHeaders().put("timestamp",
            String.valueOf(eventDate.getTimeInMillis()));
        event.getHeaders().put("hostname", "Host" + i);
        String body = "Test." + i + "." + j;
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();


      // execute sink to process the events
      sink.process();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flume.Transaction

com.github.kristofa.flume.ZipkinMetricsSink

com.github.kristofa.flume.ZipkinSpanCollectorSink

com.salesforce.phoenix.flume.sink.PhoenixSink

com.salesforce.phoenix.flume.TestRegexEventSerializer

org.apache.flume.channel.ChannelProcessor

org.apache.flume.channel.ChannelUtils

org.apache.flume.channel.FanoutChannel$wrapperTransaction

org.apache.flume.channel.file.TestFileChannel

org.apache.flume.channel.file.TestFileChannelRestart

org.apache.flume.channel.file.TestFileChannelRollback

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.