Package backtype.storm.topology

Examples of backtype.storm.topology.TopologyBuilder


    public static TopologyBuilder buildJuggaloaderTopology() {

        // see https://docs.google.com/a/nodeable.com/drawings/d/1ENR-peTco93tLFFau2A5pFi7fqpgTV7ElpSRuQka6lo/edit
       
        int PARALLELIZATION = 8;
        TopologyBuilder builder = new TopologyBuilder();
        Fields fieldsGroupingToTimeBolt = new Fields("metricAccount", "metricName");
        Fields fieldsGroupingToMessageBolt = new Fields("metricAccount", "metricName");
        Fields fieldsGroupingToPersistenceBolt = new Fields("metricAccount", "metricName");
       
        JuggaloaderPersistenceBolt persistenceBolt = new JuggaloaderPersistenceBolt();
        JuggaloaderMessageGeneratorBolt messageGenerator = new JuggaloaderMessageGeneratorBolt();
       
        builder.setSpout("spout", new JuggaloaderDummySpout(), 1);
       
        // spout-->secondsBolt
        builder.setBolt("secondsBolt", new JuggaloaderTimeBaseBolt(0), PARALLELIZATION).fieldsGrouping("spout", fieldsGroupingToTimeBolt);
       
        // secondsBolt-->minuteBolt
        builder.setBolt("minuteBolt", new JuggaloaderTimeBaseBolt(Constants.PERIOD_MINUTE), PARALLELIZATION).fieldsGrouping("secondsBolt", fieldsGroupingToTimeBolt);
       
        // minuteBolt-->hourBolt
        builder.setBolt("hourBolt", new JuggaloaderTimeBaseBolt(Constants.PERIOD_HOUR), PARALLELIZATION).fieldsGrouping("minuteBolt", fieldsGroupingToTimeBolt);

        // hourBolt-->dayBolt
        builder.setBolt("dayBolt", new JuggaloaderTimeBaseBolt(Constants.PERIOD_DAY), PARALLELIZATION).fieldsGrouping("hourBolt", fieldsGroupingToTimeBolt);
       
        // dayBolt-->weekBolt
        builder.setBolt("weekBolt", new JuggaloaderTimeBaseBolt(Constants.PERIOD_WEEK), PARALLELIZATION).fieldsGrouping("dayBolt", fieldsGroupingToTimeBolt);
       
        // weekBolt-->monthBolt
        builder.setBolt("monthBolt", new JuggaloaderTimeBaseBolt(Constants.PERIOD_MONTH), PARALLELIZATION).fieldsGrouping("weekBolt", fieldsGroupingToTimeBolt);
       
        /*
         * Tie the time-bolts to the persistence bolt.
         *
         * secondsBolt-->persistenceBolt
         * minuteBolt-->persistenceBolt
         * hourBolt-->persistenceBolt
         * dayBolt-->persistenceBolt
         * weekBolt-->persistenceBolt
         * monthBolt-->persistenceBolt
         */
        builder.setBolt("persistenceBolt", persistenceBolt, 1)
            //.fieldsGrouping("secondsBolt", fieldsGroupingToPersistenceBolt) // TODO: this fine grained not really needed
            //.fieldsGrouping("minuteBolt", fieldsGroupingToPersistenceBolt) // TODO: this fine grained not really needed
            .fieldsGrouping("hourBolt", fieldsGroupingToPersistenceBolt)
            .fieldsGrouping("dayBolt", fieldsGroupingToPersistenceBolt)
            .fieldsGrouping("weekBolt", fieldsGroupingToPersistenceBolt)
            .fieldsGrouping("monthBolt", fieldsGroupingToPersistenceBolt);
       
        /*
         * Tie the time-bolts to the message generation bolt.
         *
         * secondsBolt-->messageGenerator
         * minuteBolt-->messageGenerator
         * hourBolt-->messageGenerator
         * dayBolt-->messageGenerator
         * weekBolt-->messageGenerator
         * monthBolt-->messageGenerator
         */
        builder.setBolt("messageGenerator", messageGenerator, 1)
            .fieldsGrouping("secondsBolt", fieldsGroupingToMessageBolt)
            .fieldsGrouping("minuteBolt", fieldsGroupingToMessageBolt)
            .fieldsGrouping("hourBolt", fieldsGroupingToMessageBolt)
            .fieldsGrouping("dayBolt", fieldsGroupingToMessageBolt)
            .fieldsGrouping("weekBolt", fieldsGroupingToMessageBolt)
View Full Code Here



public class JuggaloaderTopology {

    public StormTopology createJuggaloaderTopology() {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("eventSpout", new EventSpout());
        builder.setSpout("commandSpout", new JuggaloaderCommandSpout());
        builder.setBolt("accountMetricsBolt", new AccountMetricsBolt())
                .shuffleGrouping("eventSpout", GroupingNameConstants.ACCOUNT_GROUPING_NAME);
        builder.setBolt("connectionMetricsBolt", new ConnectionMetricsBolt())
                .shuffleGrouping("eventSpout", GroupingNameConstants.CONNECTION_GROUPING_NAME);
        builder.setBolt("inventoryItemMetricsBolt", new InventoryItemMetricsBolt())
                .shuffleGrouping("eventSpout", GroupingNameConstants.INVENTORY_ITEM_GROUPING_NAME);
        builder.setBolt("userMetricsBolt", new UserMetricsBolt())
                .shuffleGrouping("eventSpout", GroupingNameConstants.USER_GROUPING_NAME);
        builder.setBolt("messageMetricsBolt", new SobaMessageMetricsBolt())
               .shuffleGrouping("eventSpout", GroupingNameConstants.MESSAGE_GROUPING_NAME);
        builder.setBolt("second", new JuggaloaderTimeBaseBolt(0))
                .fieldsGrouping("accountMetricsBolt", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("connectionMetricsBolt", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("inventoryItemMetricsBolt", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("userMetricsBolt", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("messageMetricsBolt", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("commandSpout", new Fields("metricAccount", "metricName"));
        builder.setBolt("minute", new JuggaloaderTimeBaseBolt(Constants.PERIOD_MINUTE))
                .fieldsGrouping("second", new Fields("metricAccount", "metricName"));
        builder.setBolt("hour", new JuggaloaderTimeBaseBolt(Constants.PERIOD_HOUR))
                .fieldsGrouping("minute", new Fields("metricAccount", "metricName"));
        builder.setBolt("day", new JuggaloaderTimeBaseBolt(Constants.PERIOD_DAY))
                .fieldsGrouping("hour", new Fields("metricAccount", "metricName"));
        builder.setBolt("week", new JuggaloaderTimeBaseBolt(Constants.PERIOD_WEEK))
                .fieldsGrouping("day", new Fields("metricAccount", "metricName"));
        builder.setBolt("month", new JuggaloaderTimeBaseBolt(Constants.PERIOD_MONTH))
                .fieldsGrouping("week", new Fields("metricAccount", "metricName"));
        builder.setBolt("persistence", new PersistMetricsBolt())
                .shuffleGrouping("minute")
                .shuffleGrouping("hour")
                .shuffleGrouping("day")
                .shuffleGrouping("week")
                .shuffleGrouping("month");
        // builder.setBolt("message", new JuggaloaderAnomalyGeneratorBolt()) // TODO - replace the next line with this one when SOBA-1521 is done
        builder.setBolt("message", new JuggaloaderMessageGeneratorBolt())
                .fieldsGrouping("second", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("minute", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("hour", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("day", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("week", new Fields("metricAccount", "metricName"))
                .fieldsGrouping("month", new Fields("metricAccount", "metricName"));
        return builder.createTopology();
    }
View Full Code Here

     
      //add the types
      columnTypes.add("varchar (100)");
      columnTypes.add("int");
     
      TopologyBuilder builder = new TopologyBuilder();
     
      //set the spout for the topology
      builder.setSpout("spout", new SampleSpout(), 10);

      //dump the stream data into rdbms table   
      RDBMSDumperBolt dumperBolt = new RDBMSDumperBolt(primaryKey, tableName, columnNames, columnTypes, rdbmsUrl, rdbmsUserName, rdbmsPassword);
      builder.setBolt("dumperBolt",dumperBolt, 1).shuffleGrouping("spout");

      Config conf = new Config();
      conf.setDebug(true);
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("rdbms-workflow", conf, builder.createTopology());
      Utils.sleep(10000);
      cluster.shutdown();
    }
View Full Code Here


    }
   
    public static void main(String[] args) throws Exception {
        TopologyBuilder builder = new TopologyBuilder();
       
        builder.setSpout("word", new TestWordSpout(), 10);       
        builder.setBolt("exclaim1", new ExclamationBolt(), 3)
                .shuffleGrouping("word");
        builder.setBolt("exclaim2", new ExclamationBolt(), 2)
                .shuffleGrouping("exclaim1");
               
        Config conf = new Config();
        conf.setDebug(true);
       
        if(args!=null && args.length > 0) {
            conf.setNumWorkers(3);
           
            StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
        } else {
       
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("test", conf, builder.createTopology());
            Utils.sleep(10000);
            cluster.killTopology("test");
            cluster.shutdown();   
        }
    }
View Full Code Here

    private final String topologyName;
    private final Config topologyConfig;
    private final int runtimeInSeconds;

    public RollingTopWords() throws InterruptedException {
        builder = new TopologyBuilder();
        topologyName = "slidingWindowCounts";
        topologyConfig = createTopologyConfiguration();
        runtimeInSeconds = DEFAULT_RUNTIME_IN_SECONDS;

        wireTopology();
View Full Code Here

        }
    }
   
    public static void main(String[] args) throws Exception {
       
        TopologyBuilder builder = new TopologyBuilder();
       
        builder.setSpout("spout", new RandomSentenceSpout(), 5);
       
        builder.setBolt("split", new SplitSentence(), 8)
                 .shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12)
                 .fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);

       
        if(args!=null && args.length > 0) {
            conf.setNumWorkers(3);
           
            StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
        } else {       
            conf.setMaxTaskParallelism(3);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("word-count", conf, builder.createTopology());
       
            Thread.sleep(10000);

            cluster.shutdown();
        }
View Full Code Here

        }
       
    }
   
    public static void main(String[] args) {
        TopologyBuilder builder = new TopologyBuilder();
        LocalDRPC drpc = new LocalDRPC();
       
        DRPCSpout spout = new DRPCSpout("exclamation", drpc);
        builder.setSpout("drpc", spout);
        builder.setBolt("exclaim", new ExclamationBolt(), 3)
                .shuffleGrouping("drpc");
        builder.setBolt("return", new ReturnResults(), 3)
                .shuffleGrouping("exclaim");
       
        LocalCluster cluster = new LocalCluster();
        Config conf = new Config();
        cluster.submitTopology("exclaim", conf, builder.createTopology());
       
        System.out.println(drpc.execute("exclamation", "aaa"));
        System.out.println(drpc.execute("exclamation", "bbb"));
       
    }
View Full Code Here

public class SingleJoinExample {
    public static void main(String[] args) {
        FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
        FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
       
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("gender", genderSpout);
        builder.setSpout("age", ageSpout);
        builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age")))
                .fieldsGrouping("gender", new Fields("id"))
                .fieldsGrouping("age", new Fields("id"));
       
        Config conf = new Config();
        conf.setDebug(true);
       
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("join-example", conf, builder.createTopology());
       
        for(int i=0; i<10; i++) {
            String gender;
            if(i % 2 == 0) {
                gender = "male";
View Full Code Here

    /**
     * @param args
     */
    public static void main(String[] args) {
        // TODO Auto-generated method stub
        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("signal-spout", new TestSignalSpout("test-signal-spout"));

        Config conf = new Config();
        conf.setDebug(true);

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(120000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
View Full Code Here

   
    private StormTopology createTopology(DRPCSpout spout) {
        final String SPOUT_ID = "spout";
        final String PREPARE_ID = "prepare-request";
       
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout(SPOUT_ID, spout);
        builder.setBolt(PREPARE_ID, new PrepareRequest())
                .noneGrouping(SPOUT_ID);
        int i=0;
        for(; i<_components.size();i++) {
            Component component = _components.get(i);
           
            Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
            if (i==1) {
                source.put(boltId(i-1), SourceArgs.single());
            } else if (i>=2) {
                source.put(boltId(i-1), SourceArgs.all());
            }
            IdStreamSpec idSpec = null;
            if(i==_components.size()-1 && component.bolt instanceof FinishedCallback) {
                idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
            }
            BoltDeclarer declarer = builder.setBolt(
                    boltId(i),
                    new CoordinatedBolt(component.bolt, source, idSpec),
                    component.parallelism);
           
            for(Map conf: component.componentConfs) {
                declarer.addConfigurations(conf);
            }
           
            if(idSpec!=null) {
                declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
            }
            if(i==0 && component.declarations.isEmpty()) {
                declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
            } else {
                String prevId;
                if(i==0) {
                    prevId = PREPARE_ID;
                } else {
                    prevId = boltId(i-1);
                }
                for(InputDeclaration declaration: component.declarations) {
                    declaration.declare(prevId, declarer);
                }
            }
            if(i>0) {
                declarer.directGrouping(boltId(i-1), Constants.COORDINATED_STREAM_ID);
            }
        }
       
        IRichBolt lastBolt = _components.get(_components.size()-1).bolt;
        OutputFieldsGetter getter = new OutputFieldsGetter();
        lastBolt.declareOutputFields(getter);
        Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
        if(streams.size()!=1) {
            throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
        }
        String outputStream = streams.keySet().iterator().next();
        List<String> fields = streams.get(outputStream).get_output_fields();
        if(fields.size()!=2) {
            throw new RuntimeException("Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
        }

        builder.setBolt(boltId(i), new JoinResult(PREPARE_ID))
                .fieldsGrouping(boltId(i-1), outputStream, new Fields(fields.get(0)))
                .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
        i++;
        builder.setBolt(boltId(i), new ReturnResults())
                .noneGrouping(boltId(i-1));
        return builder.createTopology();
    }
View Full Code Here

TOP

Related Classes of backtype.storm.topology.TopologyBuilder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.