Examples of SinkRunner

org.apache.flume.SinkRunner

A driver for {@linkplain Sink sinks} that polls them, attempting to{@linkplain Sink#process() process} events if any are available in the{@link Channel}.

Note that, unlike {@linkplain Source sources}, all sinks are polled.
@see org.apache.flume.Sink @see org.apache.flume.SourceRunner

Examples of org.apache.flume.SinkRunner

    params.put("processor.priority.s1", "3");
    params.put("processor.priority.s2", "2");
    params.put("processor.priority.s3", "1");
    context.putAll(params);
    Configurables.configure(group, context);
    SinkRunner runner = new SinkRunner(group.getProcessor());
    runner.start();
    Assert.assertEquals(LifecycleState.START, s1.getLifecycleState());
    Assert.assertEquals(LifecycleState.START, s2.getLifecycleState());
    Assert.assertEquals(LifecycleState.START, s3.getLifecycleState());
    for(int i = 0; i < 15; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(100);


    Assert.assertEquals(new Integer(10), s1.getWritten());
    Assert.assertEquals(new Integer(5), s2.getWritten());
    for(int i = 0; i < 50; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(100);


    Assert.assertEquals(new Integer(50), s2.getWritten());
    Assert.assertEquals(new Integer(5), s3.getWritten());
    // test rollover to recovered servers
    s2.setRemaining(20);
    for(int i = 0; i < 100; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(1000);


    Assert.assertEquals(new Integer(10), s1.getWritten());
    Assert.assertEquals(new Integer(55), s2.getWritten());
    Assert.assertEquals(new Integer(100), s3.getWritten());


    runner.stop();
    ch.stop();
  }

View Full Code Here

Examples of org.apache.flume.SinkRunner

    params.put("processor.priority.s2", "2");
    params.put("processor.priority.s3", "1");
    params.put("processor.maxpenalty", "10000");
    context.putAll(params);
    Configurables.configure(group, context);
    SinkRunner runner = new SinkRunner(group.getProcessor());
    runner.start();
    Assert.assertEquals(LifecycleState.START, s1.getLifecycleState());
    Assert.assertEquals(LifecycleState.START, s2.getLifecycleState());
    Assert.assertEquals(LifecycleState.START, s3.getLifecycleState());
    for(int i = 0; i < 15; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(100);


    Assert.assertEquals(new Integer(10), s1.getWritten());
    Assert.assertEquals(new Integer(5), s2.getWritten());
    for(int i = 0; i < 50; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(100);


    Assert.assertEquals(new Integer(50), s2.getWritten());
    Assert.assertEquals(new Integer(5), s3.getWritten());
    // test rollover to recovered servers
    s2.setRemaining(20);


    // get us past the retry time for the failed sink
    Thread.sleep(5000);


    for(int i = 0; i < 100; i++) {
      Transaction tx = ch.getTransaction();
      tx.begin();
      ch.put(EventBuilder.withBody("test".getBytes()));
      tx.commit();
      tx.close();
    }
    Thread.sleep(1000);


    Assert.assertEquals(new Integer(10), s1.getWritten());
    Assert.assertEquals(new Integer(70), s2.getWritten());
    Assert.assertEquals(new Integer(85), s3.getWritten());


    runner.stop();
    ch.stop();
  }

View Full Code Here

Examples of org.apache.flume.SinkRunner

          usedSinks.put(sink, groupName);
        }
        SinkGroup group = new SinkGroup(groupSinks);
        Configurables.configure(group, groupConf);
        conf.getSinkRunners().put(comp.getComponentName(),
            new SinkRunner(group.getProcessor()));
      }
    }
    // add any unasigned sinks to solo collectors
    for(Entry<String, Sink> entry : sinks.entrySet()) {
      if (!usedSinks.containsValue(entry.getKey())) {
        SinkProcessor pr = new DefaultSinkProcessor();
        List<Sink> sinkMap = new ArrayList<Sink>();
        sinkMap.add(entry.getValue());
        pr.setSinks(sinkMap);
        Configurables.configure(pr, new Context());
        conf.getSinkRunners().put(entry.getKey(),
            new SinkRunner(pr));
      }
    }
  }

View Full Code Here

Examples of org.apache.flume.SinkRunner

                    groupSinks.add(s);
                    usedSinks.put(sink, groupName);
                }
                SinkGroup group = new SinkGroup(groupSinks);
                Configurables.configure(group, groupConf);
                conf.getSinkRunners().put(comp.getComponentName(), new SinkRunner(group.getProcessor()));
            }
        }
        // add any unasigned sinks to solo collectors
        for (Map.Entry<String, Sink> entry : sinks.entrySet()) {
            if (!usedSinks.containsValue(entry.getKey())) {
                SinkProcessor pr = new DefaultSinkProcessor();
                List<Sink> sinkMap = new ArrayList<Sink>();
                sinkMap.add(entry.getValue());
                pr.setSinks(sinkMap);
                Configurables.configure(pr, new Context());
                conf.getSinkRunners().put(entry.getKey(), new SinkRunner(pr));
            }
        }
    }

View Full Code Here

Examples of org.apache.flume.SinkRunner

                    groupSinks.add(s);
                    usedSinks.put(sink, groupName);
                }
                final SinkGroup group = new SinkGroup(groupSinks);
                Configurables.configure(group, groupConf);
                conf.getSinkRunners().put(comp.getComponentName(), new SinkRunner(group.getProcessor()));
            }
        }
        // add any unassigned sinks to solo collectors
        for (final Map.Entry<String, Sink> entry : sinks.entrySet()) {
            if (!usedSinks.containsValue(entry.getKey())) {
                final SinkProcessor pr = new DefaultSinkProcessor();
                final List<Sink> sinkMap = new ArrayList<Sink>();
                sinkMap.add(entry.getValue());
                pr.setSinks(sinkMap);
                Configurables.configure(pr, new Context());
                conf.getSinkRunners().put(entry.getKey(), new SinkRunner(pr));
            }
        }
    }

View Full Code Here

Examples of org.apache.flume.SinkRunner

                    groupSinks.add(s);
                    usedSinks.put(sink, groupName);
                }
                SinkGroup group = new SinkGroup(groupSinks);
                Configurables.configure(group, groupConf);
                conf.getSinkRunners().put(comp.getComponentName(), new SinkRunner(group.getProcessor()));
            }
        }
        // add any unasigned sinks to solo collectors
        for (Map.Entry<String, Sink> entry : sinks.entrySet()) {
            if (!usedSinks.containsValue(entry.getKey())) {
                SinkProcessor pr = new DefaultSinkProcessor();
                List<Sink> sinkMap = new ArrayList<Sink>();
                sinkMap.add(entry.getValue());
                pr.setSinks(sinkMap);
                Configurables.configure(pr, new Context());
                conf.getSinkRunners().put(entry.getKey(), new SinkRunner(pr));
            }
        }
    }

View Full Code Here

Examples of org.apache.flume.SinkRunner

                    groupSinks.add(s);
                    usedSinks.put(sink, groupName);
                }
                final SinkGroup group = new SinkGroup(groupSinks);
                Configurables.configure(group, groupConf);
                conf.getSinkRunners().put(comp.getComponentName(), new SinkRunner(group.getProcessor()));
            }
        }
        // add any unasigned sinks to solo collectors
        for (final Map.Entry<String, Sink> entry : sinks.entrySet()) {
            if (!usedSinks.containsValue(entry.getKey())) {
                final SinkProcessor pr = new DefaultSinkProcessor();
                final List<Sink> sinkMap = new ArrayList<Sink>();
                sinkMap.add(entry.getValue());
                pr.setSinks(sinkMap);
                Configurables.configure(pr, new Context());
                conf.getSinkRunners().put(entry.getKey(), new SinkRunner(pr));
            }
        }
    }

View Full Code Here

Examples of org.apache.flume.SinkRunner

                    groupSinks.add(s);
                    usedSinks.put(sink, groupName);
                }
                final SinkGroup group = new SinkGroup(groupSinks);
                Configurables.configure(group, groupConf);
                conf.getSinkRunners().put(comp.getComponentName(), new SinkRunner(group.getProcessor()));
            }
        }
        // add any unassigned sinks to solo collectors
        for (final Map.Entry<String, Sink> entry : sinks.entrySet()) {
            if (!usedSinks.containsValue(entry.getKey())) {
                final SinkProcessor pr = new DefaultSinkProcessor();
                final List<Sink> sinkMap = new ArrayList<Sink>();
                sinkMap.add(entry.getValue());
                pr.setSinks(sinkMap);
                Configurables.configure(pr, new Context());
                conf.getSinkRunners().put(entry.getKey(), new SinkRunner(pr));
            }
        }
    }

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.