Package backtype.storm.topology

Examples of backtype.storm.topology.TopologyBuilder

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
        Map localCfg = new LinkedHashMap(conf);
        localCfg.put(ConfigurationOptions.ES_MAPPING_ID, "number");

        String target = index + "/id-write";
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("test-spout-2", new TestSpout(ImmutableList.of(doc2, doc1), new Fields("key1", "val1", "key2",
                "val2", "key3", "number")));
        builder.setBolt("es-bolt-2", new TestBolt(new EsBolt(target, localCfg))).shuffleGrouping("test-spout-2");

        MultiIndexSpoutStormSuite.run(index + "id-write", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

        RestUtils.refresh(index);
        Thread.sleep(1000);
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    public void testSimpleWriteTopology() throws Exception {
        List doc1 = Collections.singletonList(ImmutableMap.of("one", 1, "two", 2));
        List doc2 = Collections.singletonList(ImmutableMap.of("OTP", "Otopeni", "SFO", "San Fran"));

        String target = index + "/simple-write";
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("test-spout-1", new TestSpout(ImmutableList.of(doc2, doc1), new Fields("doc")));
        builder.setBolt("es-bolt-1", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-1");

        MultiIndexSpoutStormSuite.run(index + "simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

        RestUtils.refresh(index);
        assertTrue(RestUtils.exists(target));
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
        RestUtils.putData(index + "/bar",
                "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
        RestUtils.refresh(index);

        String target = "_all/foo";
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("es-spout", new TestSpout(new EsSpout(target)));
        builder.setBolt("test-bolt", new CapturingBolt()).shuffleGrouping("es-spout");

        MultiIndexSpoutStormSuite.run(index + "multi", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

        String results = RestUtils.get(target + "/_search?");
        assertThat(results, containsString("Hello"));
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    public void test1WriteIndexPattern() throws Exception {
        List doc1 = ImmutableList.of("one", 1, "two", 2, "number", 1);
        List doc2 = ImmutableList.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

        String target = index + "/write-{number}";
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("test-spout-3", new TestSpout(ImmutableList.of(doc2, doc1), new Fields("key1", "val1", "key2",
                "val2", "key3", "number")));
        builder.setBolt("es-bolt-3", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-3");

        MultiIndexSpoutStormSuite.run(index + "write-pattern", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(20));

        Thread.sleep(1000);
        RestUtils.refresh(index);
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    public void testSimpleWriteTopology() throws Exception {
        List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
        List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

        String target = index + "/json-simple-write";
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("test-spout-1", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json")));
        builder.setBolt("es-bolt-1", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-1");

        MultiIndexSpoutStormSuite.run(index + "json-simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

        RestUtils.refresh(index);
        assertTrue(RestUtils.exists(target));
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
        RestUtils.touch(index);
        RestUtils.putData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
        RestUtils.putData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
        RestUtils.refresh(index);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("es-spout", new TestSpout(new EsSpout(target)));
        builder.setBolt("test-bolt", new CapturingBolt()).shuffleGrouping("es-spout");

        MultiIndexSpoutStormSuite.run(index + "simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

        COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

        assertTrue(RestUtils.exists(target));
        String results = RestUtils.get(target + "/_search?");
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
      _ackers = 0;
    }

    try {
      for (int topoNum = 0; topoNum < _numTopologies; topoNum++) {
        TopologyBuilder builder = new TopologyBuilder();
        LOG.info("Adding in "+_spoutParallel+" spouts");
        builder.setSpout("messageSpout",
            new SOLSpout(_messageSize, _ackEnabled), _spoutParallel);
        LOG.info("Adding in "+_boltParallel+" bolts");
        builder.setBolt("messageBolt1", new SOLBolt(), _boltParallel)
            .shuffleGrouping("messageSpout");
        for (int levelNum = 2; levelNum <= _numLevels; levelNum++) {
          LOG.info("Adding in "+_boltParallel+" bolts at level "+levelNum);
          builder.setBolt("messageBolt"+levelNum, new SOLBolt(), _boltParallel)
              .shuffleGrouping("messageBolt"+(levelNum - 1));
        }

        Config conf = new Config();
        conf.setDebug(_debug);
        conf.setNumWorkers(_numWorkers);
        conf.setNumAckers(_ackers);
        if (_maxSpoutPending > 0) {
          conf.setMaxSpoutPending(_maxSpoutPending);
        }

        StormSubmitter.submitTopology(_name+"_"+topoNum, conf, builder.createTopology());
      }
      metrics(client, _messageSize, _pollFreqSec, _testRunTimeSec);
    } finally {
      //Kill it right now!!!
      KillOptions killOpts = new KillOptions();
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
        assertEquals(message, messageString);
        return true;
    }

    private Tuple generateTestTuple(Object key, Object message) {
        TopologyBuilder builder = new TopologyBuilder();
        GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
            @Override
            public Fields getComponentOutputFields(String componentId, String streamId) {
                return new Fields("key", "message");
            }
        };
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
        };
        return new TupleImpl(topologyContext, new Values(key, message), 1, "");
    }

    private Tuple generateTestTuple(Object message) {
        TopologyBuilder builder = new TopologyBuilder();
        GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
            @Override
            public Fields getComponentOutputFields(String componentId, String streamId) {
                return new Fields("message");
            }
        };
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        HBaseBolt hbase = new HBaseBolt("WordCount", mapper)
                .withConfigKey("hbase.conf");


        // wordSpout ==> countBolt ==> HBaseBolt
        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout(WORD_SPOUT, spout, 1);
        builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT);
        builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));


        if (args.length == 1) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("test", config, builder.createTopology());
            Thread.sleep(30000);
            cluster.killTopology("test");
            cluster.shutdown();
            System.exit(0);
        } else if (args.length == 2) {
            StormSubmitter.submitTopology(args[1], config, builder.createTopology());
        } else{
            System.out.println("Usage: HdfsFileTopology <hdfs url> [topology name]");
        }
    }
View Full Code Here
TOP

Related Classes of backtype.storm.topology.TopologyBuilder

  • backtype.storm.contrib.hbase.examples.HBaseCountersTopology
  • backtype.storm.contrib.hbase.examples.HBaseExampleTopology
  • backtype.storm.contrib.jms.example.ExampleJmsTopology
  • backtype.storm.contrib.signals.test.SignalTopology
  • backtype.storm.drpc.LinearDRPCTopologyBuilder
  • backtype.storm.generated.Bolt
  • backtype.storm.generated.ComponentCommon
  • backtype.storm.generated.GlobalStreamId
  • backtype.storm.generated.NullStruct
  • backtype.storm.generated.SpoutSpec
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.