Package org.apache.flume

Examples of org.apache.flume.Event

165
166
167
168
169
170
171
172
173
174
175
    try {
      txn.begin();
      String indexName = getIndexName();
      BulkRequestBuilder bulkRequest = client.prepareBulk();
      for (int i = 0; i < batchSize; i++) {
        Event event = channel.take();

        if (event == null) {
          break;
        }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  @Test
  public void testSelection() throws Exception {
    Assert.assertTrue(selector instanceof MultiplexingChannelSelector);

    Event event1 = new MockEvent();
    Map<String, String> header1 = new HashMap<String, String>();
    header1.put("myheader", "foo");// should match ch1 ch2
    event1.setHeaders(header1);

    List<Channel> reqCh1 = selector.getRequiredChannels(event1);
    Assert.assertEquals(2, reqCh1.size());
    Assert.assertTrue(reqCh1.get(0).getName().equals("ch1"));
    Assert.assertTrue(reqCh1.get(1).getName().equals("ch2"));
    List<Channel> optCh1 = selector.getOptionalChannels(event1);
    Assert.assertTrue(optCh1.size() == 1);
    //ch2 should not be there -- since it is a required channel
    Assert.assertTrue(optCh1.get(0).getName().equals("ch3"));



    Event event2 = new MockEvent();
    Map<String, String> header2 = new HashMap<String, String>();
    header2.put("myheader", "bar"); // should match ch2 ch3
    event2.setHeaders(header2);

    List<Channel> reqCh2 = selector.getRequiredChannels(event2);
    Assert.assertEquals(2, reqCh2.size());
    Assert.assertTrue(reqCh2.get(0).getName().equals("ch2"));
    Assert.assertTrue(reqCh2.get(1).getName().equals("ch3"));
    List<Channel> optCh2 = selector.getOptionalChannels(event2);
    Assert.assertTrue(optCh2.size() == 0);

    Event event3 = new MockEvent();
    Map<String, String> header3 = new HashMap<String, String>();
    header3.put("myheader", "xyz"); // should match ch1 ch2 ch3
    event3.setHeaders(header3);

    List<Channel> reqCh3 = selector.getRequiredChannels(event3);
    Assert.assertEquals(3, reqCh3.size());
    Assert.assertTrue(reqCh3.get(0).getName().equals("ch1"));
    Assert.assertTrue(reqCh3.get(1).getName().equals("ch2"));
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
  public void testDefaultBehavior() throws Exception {
    RegexHbaseEventSerializer s = new RegexHbaseEventSerializer();
    Context context = new Context();
    s.configure(context);
    String logMsg = "The sky is falling!";
    Event e = EventBuilder.withBody(Bytes.toBytes(logMsg));
    s.initialize(e, "CF".getBytes());
    List<Row> actions = s.getActions();
    assertTrue(actions.size() == 1);
    assertTrue(actions.get(0) instanceof Put);
    Put put = (Put) actions.get(0);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    String logMsg = "33.22.11.00 - - [20/May/2011:07:01:19 +0000] " +
      "\"GET /wp-admin/css/install.css HTTP/1.0\" 200 813 " +
      "\"http://www.cloudera.com/wp-admin/install.php\" \"Mozilla/5.0 (comp" +
      "atible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\"";
   
    Event e = EventBuilder.withBody(Bytes.toBytes(logMsg));
    s.initialize(e, "CF".getBytes());
    List<Row> actions = s.getActions();
    assertEquals(1, s.getActions().size());
    assertTrue(actions.get(0) instanceof Put);
   
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
145
146
   
    // Reset shared nonce to zero
    RegexHbaseEventSerializer.nonce.set(0);
    String randomString = RegexHbaseEventSerializer.randomKey;
   
    Event e1 = EventBuilder.withBody(Bytes.toBytes("body"));
    Event e2 = EventBuilder.withBody(Bytes.toBytes("body"));
    Event e3 = EventBuilder.withBody(Bytes.toBytes("body"));

    Calendar cal = mock(Calendar.class);
    when(cal.getTimeInMillis()).thenReturn(1L);
   
    s1.initialize(e1, "CF".getBytes());
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
  //it should always be mapped to the default channel(s).
  @Test
  public void testNoSelection() throws Exception {
    Assert.assertTrue(selector instanceof MultiplexingChannelSelector);

    Event noHeaderEvent = new MockEvent();

    List<Channel> reqCh1 = selector.getRequiredChannels(noHeaderEvent);
    List<Channel> optCh1 = selector.getOptionalChannels(noHeaderEvent);
    Assert.assertEquals(2, reqCh1.size());
    Assert.assertTrue(reqCh1.get(0).getName().equals("ch1"));
    Assert.assertTrue(reqCh1.get(1).getName().equals("ch3"));
    Assert.assertTrue(optCh1.size() == 0);

    Map<String, String> header2 = new HashMap<String, String>();
    header2.put("someheader", "foo");
    Event invalidHeaderEvent = new MockEvent();
    invalidHeaderEvent.setHeaders(header2);

    List<Channel> reqCh2 = selector.getRequiredChannels(invalidHeaderEvent);
    List<Channel> optCh2 = selector.getOptionalChannels(invalidHeaderEvent);
    Assert.assertEquals(2, reqCh2.size());
    Assert.assertTrue(reqCh2.get(0).getName().equals("ch1"));
    Assert.assertTrue(reqCh2.get(1).getName().equals("ch3"));
    Assert.assertTrue(optCh2.size() == 0);

    Map<String, String> header3 = new HashMap<String, String>();
    header3.put("myheader", "bar1");
    Event unmatchedHeaderEvent = new MockEvent();
    unmatchedHeaderEvent.setHeaders(header3);

    List<Channel> reqCh3 = selector.getRequiredChannels(unmatchedHeaderEvent);
    List<Channel> optCh3 = selector.getOptionalChannels(unmatchedHeaderEvent);
    Assert.assertEquals(2, reqCh3.size());
    Assert.assertTrue(reqCh3.get(0).getName().equals("ch1"));
    Assert.assertTrue(reqCh3.get(1).getName().equals("ch3"));
    Assert.assertTrue(optCh3.size() == 0);

    Map<String, String> header4 = new HashMap<String, String>();
    header4.put("myheader", "zebra");
    Event zebraEvent = new MockEvent();
    zebraEvent.setHeaders(header4);

    List<Channel> reqCh4 = selector.getRequiredChannels(zebraEvent);
    List<Channel> optCh4 = selector.getOptionalChannels(zebraEvent);
    Assert.assertEquals(2, reqCh4.size());
    Assert.assertTrue(reqCh4.get(0).getName().equals("ch1"));
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
      int batchSize = rpcClient.getBatchSize();
      List<String> lines = Lists.newLinkedList();
      while (!(lines = reader.readLines(batchSize)).isEmpty()) {
        List<Event> eventBuffer = Lists.newArrayList();
        for (String line : lines) {
          Event event = EventBuilder.withBody(line, Charsets.UTF_8);
          setHeaders(event);
          eventBuffer.add(event);
          sentBytes += event.getBody().length;
          sent++;

          long now = System.currentTimeMillis();
          if (now >= lastCheck + 5000) {
            logger.debug("Packed {} bytes, {} events", sentBytes, sent);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    source.start();
    Transaction transaction = channel.getTransaction();

    transaction.begin();
    Event event;

    FileOutputStream outputStream = new FileOutputStream(
        "/tmp/flume-execsource." + Thread.currentThread().getId());

    while ((event = channel.take()) != null) {
      outputStream.write(event.getBody());
      outputStream.write('\n');
    }

    outputStream.close();
    transaction.commit();
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
        SpoolDirectorySourceConfigurationConstants.BUFFER_MAX_LINE_LENGTH,
        SpoolDirectorySourceConfigurationConstants.DEFAULT_BUFFER_MAX_LINE_LENGTH);
  }

  private Event createEvent(String lineEntry, String filename) {
    Event out = EventBuilder.withBody(lineEntry.getBytes());
    if (fileHeader) {
      out.getHeaders().put(fileHeaderKey, filename);
    }
    return out;
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    Configurables.configure(channel, new Context());
    sink.setChannel(channel);
    sink.start();
    Transaction tx = channel.getTransaction();
    tx.begin();
    Event e = EventBuilder.withBody(
        Bytes.toBytes(valBase));
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();
    HTable table = new HTable(testUtility.getConfiguration(), tableName);
    byte[][] results = getResults(table, 1);
    byte[] out = results[0];
    Assert.assertArrayEquals(e.getBody(), out);
    out = results[1];
    Assert.assertArrayEquals(Longs.toByteArray(1), out);
    testUtility.deleteTable(tableName.getBytes());
  }
View Full Code Here
TOP

Related Classes of org.apache.flume.Event

  • com.salesforce.phoenix.flume.TestRegexEventSerializer
  • org.apache.flume.agent.embedded.TestEmbeddedAgent
  • org.apache.flume.agent.embedded.TestEmbeddedAgentEmbeddedSource
  • org.apache.flume.api.NettyAvroRpcClient
  • org.apache.flume.channel.file.FileChannel$FileBackedTransaction
  • org.apache.flume.channel.file.TestFileChannel
  • org.apache.flume.channel.file.TestUtils
  • org.apache.flume.channel.jdbc.BaseJdbcChannelProviderTest
  • org.apache.flume.channel.jdbc.TestJdbcChannelProvider
  • org.apache.flume.channel.MemoryChannel$MemoryTransaction
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.