Package de.jetwick.es

Examples of de.jetwick.es.ElasticTweetSearch


        Runtime.getRuntime().addShutdownHook(new Thread(runOnExit));

        Module module = new DefaultModule();
        Injector injector = Guice.createInjector(module);
        TwitterSearch tws = injector.getInstance(TwitterSearch.class);
        ElasticTweetSearch tweetSearch = injector.getInstance(ElasticTweetSearch.class);

        // WAIT UNTIL AVAILABLE
        tweetSearch.waitUntilAvailable(10000);
       
        ElasticUserSearch userSearch = injector.getInstance(ElasticUserSearch.class);
        ElasticTagSearch tagSearch = injector.getInstance(ElasticTagSearch.class);
        Configuration cfg = injector.getInstance(Configuration.class);

        // 1. every producer has a separate queue (with a different capacity) to feed TweetConsumer:
        //      TProd1 -- queue1 --\
        //      TProd2 -- queue2 ---> TweetConsumer
        //      ...
       
        // 2. TweetConsumer polls N elements from every queue and feeds the results
        //    into the resolver - see GenericUrlResolver.
       
        // 4. Via ElasticTweetSearch:s commit listener the URL:s of tweets will be       
        //    resolved -
        //    For every URL an article is created and feeded into the article index

        TweetConsumer twConsumer = injector.getInstance(TweetConsumer.class);
        twConsumer.setUncaughtExceptionHandler(excHandler);

        GenericUrlResolver resolver = injector.getInstance(GenericUrlResolver.class);       
        resolver.start();
        int queueCapacity = cfg.getUrlResolverInputQueueSize();
        // feeding consumer via twitter search (or offline fake)
        TweetProducer twProducer = injector.getInstance(TweetProducer.class);               
        twProducer.setTwitterSearch(tws);
        twProducer.setUserSearch(userSearch);
        twProducer.setTagSearch(tagSearch);
        twProducer.setQueue(twConsumer.register("producer-search", queueCapacity, 100));

        // feeding consumer via twitter keyword stream (gets keywords from tagindex)
        TweetProducerViaStream producerViaStream = injector.getInstance(TweetProducerViaStream.class);
        producerViaStream.setQueue(twConsumer.register("producer-stream", queueCapacity, 120));
        producerViaStream.setTwitterSearch(tws);
        producerViaStream.setTagSearch(tagSearch);
        producerViaStream.setUncaughtExceptionHandler(excHandler);
        producerViaStream.setTweetsPerSecLimit(cfg.getTweetsPerSecLimit());

        // feeding consumer from tweets of friends (of registered users)
        TweetProducerViaUsers producerFromFriends = injector.getInstance(TweetProducerViaUsers.class);
        producerFromFriends.setQueue(twConsumer.register("producer-friends", queueCapacity, 100));
        producerFromFriends.setTwitterSearch(tws);
        producerFromFriends.setUserSearch(userSearch);
        producerFromFriends.setUncaughtExceptionHandler(excHandler);

        // feeding consumer from UI       
        RMIServer rmiServer = injector.getInstance(RMIServer.class);
        rmiServer.setQueue(twConsumer.register("producer-rmi", queueCapacity, 20));
        Thread rmiServerThread = rmiServer.createThread();
 
        // configure tweet index to call UrlResolver after feeding of a tweet       
        tweetSearch.setRemoveOlderThanDays(cfg.getTweetSearchRemoveDays());
        tweetSearch.setBatchSize(cfg.getTweetSearchBatch());               

        Thread twProducerThread = new Thread(twProducer, "tweet-producer");
        twProducerThread.setUncaughtExceptionHandler(excHandler);
        twProducerThread.start();
View Full Code Here


    public void installSearchModule() {
        // TODO shouldn't fail when node is not available!!??
        Client client = AbstractElasticSearch.createClient(ElasticNode.CLUSTER,
                config.getTweetSearchUrl(), ElasticNode.PORT);

        ElasticTweetSearch tweetSearch = new ElasticTweetSearch(client);
        try {
            tweetSearch.nodeInfo();
        } catch (Exception ex) {
            logger.warn("Problem to get node info:" + ex.getMessage());
        }
        bind(ElasticTweetSearch.class).toInstance(tweetSearch);
View Full Code Here

        Provider<RMIClient> rmiProvider = injector.getProvider(RMIClient.class);
        Configuration cfg = injector.getInstance(Configuration.class);
        TwitterSearch twSearch = injector.getInstance(TwitterSearch.class);
        twSearch.initTwitter4JInstance(cfg.getTwitterSearchCredits().getToken(),
                cfg.getTwitterSearchCredits().getTokenSecret(), true);
        ElasticTweetSearch fromUserSearch = new ElasticTweetSearch(injector.getInstance(Configuration.class));
        JetwickQuery query = new TweetQuery().addFilterQuery(ElasticTweetSearch.UPDATE_DT, "[* TO *]");
        // TODO set facetlimit to 2000
        query.addFacetField("user").setSize(0);
        SearchResponse rsp = fromUserSearch.query(query);

        TermsFacet tf = (TermsFacet) rsp.getFacets().facet("user");
        logger.info("found: " + tf.entries().size() + " users with the specified criteria");
        int SLEEP = 30;
        int counter = 0;
View Full Code Here

        // TODO send via RMI
    }

    public void fillFrom(final String fromUrl) {
        ElasticTweetSearch fromTweetSearch = new ElasticTweetSearch(fromUrl);
        JetwickQuery query = new TweetQuery();
        long maxPage = 1;
        int hitsPerPage = 300;
        Set<JUser> users = new LinkedHashSet<JUser>();
        Runnable optimizeOnExit = new Runnable() {

            @Override
            public void run() {
                userSearch.refresh();
                logger.info(userCounter + " users pushed to default tweet search from " + fromUrl);
            }
        };
        Runtime.getRuntime().addShutdownHook(new Thread(optimizeOnExit));

        for (int page = 0; page < maxPage; page++) {
            query.attachPagability(page, hitsPerPage);
            users.clear();

            SearchResponse rsp;
            try {
                rsp = fromTweetSearch.query(users, query);
            } catch (Exception ex) {
                logger.warn("Error while searching!", ex);
                continue;
            }
            if (maxPage == 1) {
View Full Code Here

    }

    @Test
    public void testNormalSearch() throws Exception {
//        setUp();
        ElasticTweetSearch search = getInstance(ElasticTweetSearch.class);
        JetwickQuery query = new TweetQuery("timetabling");
        tester.startPage(new TweetSearchPage(query));
        tester.assertNoErrorMessage();

        verify(search).query(new LinkedHashSet<JUser>(), query);
View Full Code Here

                        retweeted.add(twitterId);
                        return null;
                    }
                };

                tweetSearch = new ElasticTweetSearch() {

                    @Override
                    public SearchResponse query(JetwickQuery query) {
                        return null;
                    }
View Full Code Here

        Thread.UncaughtExceptionHandler excHandler = createExceptionMapHandler(exceptionMap);

        // fill DB with one default tag
        tagSearchTester.getSearch().bulkUpdate(Arrays.asList(new JTag("java")), tagSearchTester.getSearch().getIndexName(), true);

        ElasticTweetSearch tweetSearch = tweetSearchTester.getSearch();
        ElasticUserSearch userSearch = userSearchTester.getSearch();
        TwitterSearch tws = new TwitterSearch() {

            @Override
            public boolean isInitialized() {
                return true;
            }

            @Override
            public long search(String q, Collection<JTweet> result, int tweets, long lastMillis) {
                JUser u = new JUser("timetabling");
                JTweet tw1 = new JTweet(1L, "... Egypt. http://apne.ws/dERa4A - XY #tEst", u);
                result.add(tw1);
                return lastMillis;
            }

            @Override
            public List<JTweet> getTweets(JUser user, Collection<JUser> users, int twPerPage) {
                return Collections.EMPTY_LIST;
            }
        };

        TweetConsumer tweetConsumer = getInstance(TweetConsumer.class);
        tweetConsumer.setUncaughtExceptionHandler(excHandler);
        tweetConsumer.start();

        TweetProducer tweetProducer = getInstance(TweetProducer.class);
        tweetProducer.setTwitterSearch(tws);
        tweetProducer.setUserSearch(userSearch);
        tweetProducer.setTagSearch(tagSearchTester.getSearch());
        tweetProducer.setQueue(tweetConsumer.register("tweet-producer", Integer.MAX_VALUE, 100));

        Thread tweetProducerThread = new Thread(tweetProducer);
        tweetProducerThread.setUncaughtExceptionHandler(excHandler);
        tweetProducerThread.start();
        // wait so let consumer consume
        Thread.sleep(500);

        tweetConsumer.interrupt();
        tweetProducerThread.interrupt();
        checkExceptions(exceptionMap);

        tweetSearch.forceEmptyQueueAndRefresh();
        List<JTweet> res = tweetSearch.searchTweets(new TweetQuery().addFilterQuery(ElasticTweetSearch.USER, "timetabling"));
        assertEquals(1, res.size());
        assertEquals(1, res.get(0).getUrlEntries().size());
        // no resolving for now
        assertTrue(res.get(0).getUrlEntries().iterator().next().getResolvedUrl().equals("http://apne.ws/dERa4A"));
    }
View Full Code Here

        // fill DB with one default tag
        tagSearchTester.getSearch().bulkUpdate(Arrays.asList(new JTag("java")), tagSearchTester.getSearch().getIndexName(), true);

        ElasticUserSearch userSearch = userSearchTester.getSearch();
        ElasticTweetSearch tweetSearch = tweetSearchTester.getSearch();

        // already existing tweets must not harm
        tweetSearch.store(new JTweet(3L, "duplication tweet", new JUser("tmp")), true);
        TwitterSearch tws = new TwitterSearch() {

            @Override
            public boolean isInitialized() {
                return true;
            }

            @Override
            public long search(String q, Collection<JTweet> result, int tweets, long lastMillis) {
                JUser u = new JUser("timetabling");
                JTweet tw1 = new JTweet(1L, "test", u);
                result.add(tw1);

                tw1 = new JTweet(2L, "java test", u);
                result.add(tw1);

                // this tweet will be ignored and so it won't be indexed!
                tw1 = new JTweet(3L, "duplicate tweet", new JUser("anotheruser"));
                result.add(tw1);

                tw1 = new JTweet(4L, "reference a user: @timetabling", new JUser("user3"));
                result.add(tw1);

                assertEquals(4, result.size());
                return lastMillis;
            }

            @Override
            public List<JTweet> getTweets(JUser user, Collection<JUser> users, int twPerPage) {
                return Collections.EMPTY_LIST;
            }
        };

        TweetConsumer tweetConsumer = getInstance(TweetConsumer.class);
        tweetConsumer.setUncaughtExceptionHandler(excHandler);
        tweetConsumer.start();

        TweetProducer tweetProducer = getInstance(TweetProducer.class);
        tweetProducer.setTwitterSearch(tws);
        tweetProducer.setUserSearch(userSearch);
        tweetProducer.setTagSearch(tagSearchTester.getSearch());
        tweetProducer.setQueue(tweetConsumer.register("tweet-producer", Integer.MAX_VALUE, 100));

        Thread tweetProducerThread = new Thread(tweetProducer);
        tweetProducerThread.setUncaughtExceptionHandler(excHandler);
        tweetProducerThread.start();
        Thread.sleep(500);

        tweetConsumer.interrupt();
        checkExceptions(exceptionMap);
        tweetSearch.forceEmptyQueueAndRefresh();

        Set<JUser> users = new LinkedHashSet<JUser>();
        tweetSearch.query(users, new TweetQuery().addFilterQuery(ElasticTweetSearch.USER, "timetabling"));
        assertEquals(2, users.iterator().next().getOwnTweets().size());

        List<JUser> res = new ArrayList<JUser>();
        tweetSearch.query(res, new TweetQuery("java"));
        assertEquals(1, res.size());

        Collection<JTweet> coll = tweetSearch.searchTweets(new TweetQuery("duplicate"));
        assertEquals(1, coll.size());
        assertEquals("duplication tweet", coll.iterator().next().getText());

        coll = tweetSearch.searchTweets(new TweetQuery("duplication"));
        assertEquals(1, coll.size());
        assertEquals("duplication tweet", coll.iterator().next().getText());
    }
View Full Code Here

            public List<JTweet> getTweets(JUser user, Collection<JUser> users, int twPerPage) {
                return Collections.EMPTY_LIST;
            }
        };

        ElasticTweetSearch tweetSearch = getInstance(ElasticTweetSearch.class);
        GenericUrlResolver resolver = getInstance(GenericUrlResolver.class);
        tweetSearch.addListener(resolver);
       
        TweetConsumer tweetConsumer = getInstance(TweetConsumer.class);
        tweetConsumer.setUncaughtExceptionHandler(excHandler);
        tweetConsumer.start();

        TweetProducer tweetProducer = getInstance(TweetProducer.class);
        tweetProducer.setTwitterSearch(tws);
        tweetProducer.setUserSearch(getInstance(ElasticUserSearch.class));
        tweetProducer.setTagSearch(tagSearchTester.getSearch());
        tweetProducer.setQueue(tweetConsumer.register("tweet-producer", Integer.MAX_VALUE, 100));

        Thread tweetProducerThread = new Thread(tweetProducer);
        tweetProducerThread.setUncaughtExceptionHandler(excHandler);
        tweetProducerThread.start();
        // let tweetconsumer do its work
        Thread.sleep(500);
               
        tweetSearch.forceEmptyQueueAndRefresh();
        List<JTweet> tweets = tweetSearch.searchTweets(new TweetQuery().setSort(ElasticTweetSearch.RT_COUNT, "desc"));
        assertEquals(2, tweets.size());
        assertEquals(1, tweets.get(0).getRetweetCount());
        assertEquals(0, tweets.get(1).getRetweetCount());       
    }
View Full Code Here

                bind(ElasticUserSearch.class).toInstance(mockUserSearch(userSearch));

                ElasticTagSearch tagSearch = mock(ElasticTagSearch.class);
                bind(ElasticTagSearch.class).toInstance(tagSearch);

                ElasticTweetSearch twSearch = mock(ElasticTweetSearch.class);

                // mock this hit/result too!
                //new InternalSearchHit(1, "1", "tweet", source, fields);
                InternalSearchResponse iRsp2 = new InternalSearchResponse(
                        new InternalSearchHits(new InternalSearchHit[0], 0, 0), new InternalFacets(new ArrayList()), true);
                when(twSearch.query((Collection<JUser>) any(), (TweetQuery) any())).
                        thenReturn(new SearchResponse(iRsp2, "", 4, 4, 1L, new ShardSearchFailure[0]));

                bind(ElasticTweetSearch.class).toInstance(twSearch);
            }
View Full Code Here

TOP

Related Classes of de.jetwick.es.ElasticTweetSearch

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.