Examples of com.cloudera.flume.core.Event

com.cloudera.flume.core.Event
This is the abstract class for events in flume.

    mFieldMap = Collections.unmodifiableMap(mFieldMap);
  }


  @Override
  public void takeEvent(EventWrapper e) throws IOException, InterruptedException {
    Event event = e.getEvent();


    // Determine which stream the event is from; this determines which map we
    // place the event in, and which map we check for candidate join matches.
    String streamName = e.getAttr(STREAM_NAME_ATTR);
    if (null == streamName) {
      // We don't know which stream this came from. Don't process it.
      LOG.warn("Got event with no " + STREAM_NAME_ATTR + " attribute!");
      return;
    }


    WindowedHashMap<Object, EventWrapper, Long> insertMap; // Map where we insert this event.
    WindowedHashMap<Object, EventWrapper, Long> joinMap; // Map we pull join candidates from.
    TypedField keyField; // The field to grab from the event wrapper.
    boolean isLeft;


    if (streamName.equals(mLeftName)) {
      insertMap = mLeftMap;
      joinMap = mRightMap;
      keyField = mLeftKey;
      isLeft = true;
    } else if (streamName.equals(mRightName)) {
      insertMap = mRightMap;
      joinMap = mLeftMap;
      keyField = mRightKey;
      isLeft = false;
    } else {
      // Not from either stream?
      LOG.warn("Got event with unexpected " + STREAM_NAME_ATTR + "=" + streamName);
      return; // Don't know what to do with this.
    }


    // Look up elements from the opposite map to determine what joins we can perform.
    Object key = e.getField(keyField);
    if (null == key) {
      // The key field is null; this will not match to anything in an inner join.
      return;
    }


    assert mTimeSpan.isRelative;
    long curTime = event.getTimestamp();
    Long lo;
    Long hi;


    if (isLeft) {
      // If this event is from the left stream, calculate the relative time interval normally. 
      lo = curTime + mTimeSpan.lo;
      hi = curTime + mTimeSpan.hi;
    } else {
      // If this event is from the right stream, use the "mirror image" of the timespan.
      // "RANGE INTERVAL 10 MINUTES PRECEDING" actually means, join with the /next/ 10
      // minutes of data from this perspective.
      lo = curTime - mTimeSpan.hi;
      hi = curTime - mTimeSpan.lo;
    }


    LOG.debug("Working on key: " + key + ", isLeft=" + isLeft);
    LOG.debug("Timestamp=" + curTime + ", interval=" + lo + ", " + hi);


    // Join with all the events in the window.
    List<EventWrapper> joinEvents = joinMap.getRange(key, lo, hi, isLeft, !isLeft);
    for (EventWrapper joinWrapper : joinEvents) {
      CompositeEvent outEvent = new CompositeEvent(mFieldMap,
          event.getPriority(), event.getTimestamp(), event.getNanos(), event.getHost());
      CompositeEventWrapper outWrapper = new CompositeEventWrapper();
      if (isLeft) {
        outEvent.add(e);
        outEvent.add(joinWrapper);
      } else {

View Full Code Here

      FlowElementContext context = getContext();
      try {
        // Iterate over all the input events, and wrap them in
        // a parsing EventWrapper; advance these to the output.
        while (iter.hasNext()) {
          Event rawEvent = iter.next();
          rawEvent.set(STREAM_NAME_ATTR, mStreamSymbol.getName().getBytes());
          EventWrapper wrapper = new ParsingEventWrapper(mStreamSymbol.getEventParser(),
              mFieldNames);
          wrapper.reset(rawEvent);
          context.emit(wrapper);
        }

View Full Code Here

  public NoNlSynthSource(long count, int size) {
    super(count, size);
  }


  public Event next() throws IOException {
    Event e = super.next();
    if (e == null)
      return null;


    // NOTE: this is a reference to the body. and will be modified
    byte[] body = e.getBody();
    for (int i = 0; i < body.length; i++) {
      if (body[i] == '\n')
        body[i] = ' ';
    }

View Full Code Here

        "Need to open source before reading from it");
    String s = in.readLine();
    if (s == null)
      return null;


    Event e = new EventImpl(s.getBytes());
    updateEventProcessingStats(e);
    return e;
  }

View Full Code Here

    Preconditions.checkState(rd != null, "Next on unopened sink!");
    String s = rd.readLine();
    if (s == null) {
      return null; // end of stream
    }
    Event e = new EventImpl(s.getBytes(CharEncUtils.RAW));
    updateEventProcessingStats(e);
    return e;
  }

View Full Code Here

      while (true) {
        String s2 = raf.readLine();
        if (s2 == null) {
          // reached the last line
          prevLine = null;
          Event e = new EventImpl(builder.toString().getBytes(), d.getTime(),
              Priority.valueOf(prio), nanos, host,
              new HashMap<String, byte[]>());
          return e;
        }


        // valid line, need to get more lines.
        Matcher m2 = l4jPat.matcher(s2);
        if (m2.matches()) {
          // a new matching line? event finished, save line for next round.
          Event e = new EventImpl(builder.toString().getBytes(), d.getTime(),
              Priority.valueOf(prio), nanos, host,
              new HashMap<String, byte[]>());
          prevLine = s2;
          return e;
        }

View Full Code Here


  @Override
  public Event next() throws IOException {


    try {
      Event e = q.take();
      updateEventProcessingStats(e);
      return e;
    } catch (InterruptedException e) {
      e.printStackTrace();
      throw new IOException("Waiting for queue element was interrupted! " + e);
    }
  }

View Full Code Here

    PrioritizedThriftEventSource src = new PrioritizedThriftEventSource(conf
        .getCollectorPort());


    try {
      src.open();
      Event e;
      e = src.next();
      while (e != null) {
        System.out.println(e);
        e = src.next();
      }

View Full Code Here

  @Override
  public Event next() throws IOException {
    try {
      while (!done) {
        // This blocks on the synchronized queue until a new event arrives.
        Event e = sync.poll(100, TimeUnit.MILLISECONDS);
        if (e == null)
          continue; // nothing there, retry.
        updateEventProcessingStats(e);
        return e;
      }

View Full Code Here

        .getCollectorPort());
    try {
      sink.open();


      for (int i = 0; i < 100; i++) {
        Event e = new EventImpl(("This is a test " + i).getBytes());
        sink.append(e);
        Thread.sleep(200);
      }
    } catch (IOException e) {
      e.printStackTrace();
    } catch (InterruptedException e) {
      e.printStackTrace();
    }


  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.core.Event

com.cloudera.flume.agent.diskfailover.DiskFailoverSource

com.cloudera.flume.agent.diskfailover.NaiveFileFailoverManager$StateChangeDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverDeco

com.cloudera.flume.agent.diskfailover.TestDiskFailoverManager

com.cloudera.flume.agent.diskfailover.TestDiskFailoverSource

com.cloudera.flume.agent.durability.NaiveFileWALManager$StateChangeDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALDeco

com.cloudera.flume.agent.durability.TestNaiveFileWALHandles

com.cloudera.flume.agent.durability.TestNaiveFileWALManager

com.cloudera.flume.agent.durability.TestNaiveFileWALSource

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.