Examples of backtype.storm.Config

backtype.storm.Config
Topology configs are specified as a plain old map. This class provides a convenient way to create a topology config map by providing setter methods for all the configs that can be set. It also makes it easier to do things like add serializations.
This class also provides constants for all the configurations possible on a Storm cluster and Storm topology. Default values for these configs can be found in defaults.yaml.

Note that you may put other configurations in any of the configs. Storm will ignore anything it doesn't recognize, but your topologies are free to make use of them by reading them in the prepare method of Bolts or the open method of Spouts. .

        return false;
    }


    @Override
    public Map<String, Object> getComponentConfiguration() {
        Config ret = new Config();
        ret.setMaxTaskParallelism(1);
        ret.registerSerialization(TransactionAttempt.class);
        return ret;
    }

View Full Code Here

    static final String SENTENCE_SPOUT_ID = "sentence-spout";
    static final String BOLT_ID = "my-bolt";
    static final String TOPOLOGY_NAME = "test-topology";


    public static void main(String[] args) throws Exception {
        Config config = new Config();
        config.setNumWorkers(1);


        SentenceSpout spout = new SentenceSpout();


        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);


        // rotate files when they reach 5MB
        FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);


        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/foo/")
                .withExtension(".txt");






        // use "|" instead of "," for field delimiter
        RecordFormat format = new DelimitedRecordFormat()
                .withFieldDelimiter("|");


        Yaml yaml = new Yaml();
        InputStream in = new FileInputStream(args[1]);
        Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
        in.close();
        config.put("hdfs.config", yamlConf);


        HdfsBolt bolt = new HdfsBolt()
                .withConfigKey("hdfs.config")
                .withFsUrl(args[0])
                .withFileNameFormat(fileNameFormat)

View Full Code Here

    static final String SENTENCE_SPOUT_ID = "sentence-spout";
    static final String BOLT_ID = "my-bolt";
    static final String TOPOLOGY_NAME = "test-topology";


    public static void main(String[] args) throws Exception {
        Config config = new Config();
        config.setNumWorkers(1);


        SentenceSpout spout = new SentenceSpout();


        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);

View Full Code Here

    builder.setBolt("statsPrinterBolt:perceptron", new StatsPrinterBolt("perceptron")).shuffleGrouping("statistics:perceptron");
    builder.setBolt("statsWriterBolt:perceptron", new StatsWriterBolt("perceptron", resultsFolder)).shuffleGrouping("statistics:perceptron");*/
    
    
    //Create the configuration object
    Config conf = new Config();
    
    //Create a local cluster
    LocalCluster cluster = new LocalCluster();
    
    //Submit the topology to the cluster for execution

View Full Code Here

          LOG.info("Adding in "+_boltParallel+" bolts at level "+levelNum);
          builder.setBolt("messageBolt"+levelNum, new SOLBolt(), _boltParallel)
              .shuffleGrouping("messageBolt"+(levelNum - 1));
        }


        Config conf = new Config();
        conf.setDebug(_debug);
        conf.setNumWorkers(_numWorkers);
        conf.setNumAckers(_ackers);
        if (_maxSpoutPending > 0) {
          conf.setMaxSpoutPending(_maxSpoutPending);
        }


        StormSubmitter.submitTopology(_name+"_"+topoNum, conf, builder.createTopology());
      }
      metrics(client, _messageSize, _pollFreqSec, _testRunTimeSec);

View Full Code Here

        return true;
    }


    private Tuple generateTestTuple(Object key, Object message) {
        TopologyBuilder builder = new TopologyBuilder();
        GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
            @Override
            public Fields getComponentOutputFields(String componentId, String streamId) {
                return new Fields("key", "message");
            }
        };

View Full Code Here

        return new TupleImpl(topologyContext, new Values(key, message), 1, "");
    }


    private Tuple generateTestTuple(Object message) {
        TopologyBuilder builder = new TopologyBuilder();
        GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
            @Override
            public Fields getComponentOutputFields(String componentId, String streamId) {
                return new Fields("message");
            }
        };

View Full Code Here

    private static final String COUNT_BOLT = "COUNT_BOLT";
    private static final String HBASE_BOLT = "HBASE_BOLT";




    public static void main(String[] args) throws Exception {
        Config config = new Config();


        Map<String, Object> hbConf = new HashMap<String, Object>();
        if(args.length > 0){
            hbConf.put("hbase.rootdir", args[0]);
        }
        config.put("hbase.conf", hbConf);


        WordSpout spout = new WordSpout();
        WordCounter bolt = new WordCounter();


        SimpleHBaseMapper mapper = new SimpleHBaseMapper()

View Full Code Here

        stream.each(new Fields("word","columnValue"), new PrintFunction(), new Fields());
        return topology.build();
    }


    public static void main(String[] args) throws Exception {
        Config conf = new Config();
        conf.setMaxSpoutPending(5);
        if (args.length == 1) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
            Thread.sleep(60 * 1000);
            cluster.killTopology("wordCounter");
            cluster.shutdown();
            System.exit(0);
        }
        else if(args.length == 2) {
            conf.setNumWorkers(3);
            StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
        } else{
            System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
        }
    }

View Full Code Here


    wireTopology();
  }


  private static Config createTopologyConfiguration() {
    Config conf = new Config();
    conf.setDebug(true);
    return conf;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of backtype.storm.Config

AbstractStormWithKafkaTest

backtype.storm.testing.MemoryTransactionalSpout

backtype.storm.testing.OpaqueMemoryTransactionalSpout

backtype.storm.transactional.TransactionalSpoutCoordinator

com.alibaba.jstorm.cluster.StormConfig

com.alipay.dw.jstorm.example.drpc.ReachTopology

com.alipay.dw.jstorm.example.sequence.SequenceTopologyTool

com.alipay.dw.jstorm.transcation.TransactionalGlobalCount

com.digitalpebble.storm.crawler.bolt.FetcherBolt

com.hmsonline.storm.cassandra.bolt.CassandraBoltTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.