Package de.jetwick.data

Examples of de.jetwick.data.JTag


                }

                findNewTagsTime = System.currentTimeMillis();
            }

            JTag tag = tags.poll();
            long lastMillis = tag.getLastMillis();
            if (tag != null && tag.nextQuery()) {
                String term = tag.getTerm();
                if (term == null) {
                    // TODO use user search later on
                    logger.warn("TODO skipping tags with empty terms for now:" + tag);
                    continue;
                }

                if (term.isEmpty() || JetwickQuery.containsForbiddenChars(term))
                    continue;

                float waitInSeconds = 1f;
                try {
                    int pages = tag.getPages();
                    tmpColl.clear();
                    long newMaxCreateTime = twSearch.search(term + " " + TwitterSearch.LINK_FILTER, tmpColl, pages * 100, 0);

                    // calc tweets per sec with 'floating mean'
                    double lastTweetsPerSec = tag.getTweetsPerSec();
                    int newTweets = guessNewTweets(tmpColl, tag.getMaxCreateTime());
                    lastTweetsPerSec = lastTweetsPerSec + newTweets / ((System.currentTimeMillis() - lastMillis) / 1000.0);
                    tag.setTweetsPerSec(lastTweetsPerSec / 2);
                    tag.setMaxCreateTime(newMaxCreateTime);
                    logger.info("searched: " + tag + "\t=> tweets:" + tmpColl.size() + "\t newTweets:" + newTweets);
                    for (JTweet tw : tmpColl) {
                        try {
                            resultTweets.put(tw.setFeedSource("search:" + term));
                        } catch (InterruptedException ex) {
View Full Code Here


                for (String tmpTerm : termAsStr.split(" OR ")) {
                    if (Helper.isEmpty(termAsStr) || JetwickQuery.containsForbiddenChars(termAsStr))
                        continue;

                    JTag tag = tmpTags.get(tmpTerm);
                    if (tag == null) {
                        tag = tagSearch.findByTerm(tmpTerm);
                        if (tag == null)
                            tag = new JTag(tmpTerm);
                        tmpTags.put(tmpTerm, tag);
                        counter++;
                    }
                }
            }
View Full Code Here

        if (!getMySession().hasLoggedIn()) {
            if (q.getQuery().startsWith("#") || q.getQuery().startsWith("%"))
                return;
        }

        JTag t = new JTag(q.getQuery());
        if (!Helper.isEmpty(q.extractUserName()))
            t.setUser(q.extractUserName());

        tagindexProvider.get().queueObject(t, true);
    }
View Full Code Here

        tagSearchTester.tearDown();
    }

    @Test
    public void testUpdateAndInitTag() {
        tagSearchTester.getSearch().queueObject(new JTag("java").setTweetsPerSec(0.5));
        tagSearchTester.getSearch().queueObject(new JTag("google").setTweetsPerSec(0.5));
        tagSearchTester.getSearch().forceCleanTagQueueAndRefresh();
       
        twProd.setNewStreamInterval(30 * 1000);
        twProd.run();       
       
View Full Code Here

    }

    @Test
    public void testUpdateAndInitTag() {
        // make sure that updateTag is in a transaction               
        twProd.updateTag(new JTag("test"), 6);
        tagSearchTester.getSearch().forceCleanTagQueueAndRefresh();
        assertTrue(tagSearchTester.getSearch().findByTerm("test").getQueryInterval() < 10 * JTag.DEFAULT_Q_I);
    }
View Full Code Here

        assertTrue(tagSearchTester.getSearch().findByTerm("test").getQueryInterval() < 10 * JTag.DEFAULT_Q_I);
    }

    @Test
    public void testInitTagsNoException() {
        twProd.updateTag(new JTag("test"), 6);
        ElasticUserSearch uSearch = mock(ElasticUserSearch.class);
        ElasticTagSearch tagSearch = mock(ElasticTagSearch.class);
        when(uSearch.getQueryTerms()).thenReturn(Arrays.asList("test OR pest"));
        when(tagSearch.findSorted(0, 1000)).thenReturn(Arrays.asList(new JTag("solr OR lucene")));
        twProd.setUserSearch(uSearch);
        twProd.setTagSearch(tagSearch);
        Collection<JTag> tags = twProd.initTags();

        assertEquals(3, tags.size());
        String str = "";
        for (JTag tag : tags) {
            str += tag.getTerm();
            twProd.updateTag(tag, 5);
        }
        assertTrue(str.contains("test"));
        assertTrue(str.contains("pest"));
        assertTrue(str.contains("solr"));
        assertTrue(str.contains("lucene"));

        twProd.updateTag(new JTag("anotherone"), 6);
    }
View Full Code Here

    public void testUrlResolving() throws Exception {
        final Map<Thread, Throwable> exceptionMap = new HashMap<Thread, Throwable>();
        Thread.UncaughtExceptionHandler excHandler = createExceptionMapHandler(exceptionMap);

        // fill DB with one default tag
        tagSearchTester.getSearch().bulkUpdate(Arrays.asList(new JTag("java")), tagSearchTester.getSearch().getIndexName(), true);

        ElasticTweetSearch tweetSearch = tweetSearchTester.getSearch();
        ElasticUserSearch userSearch = userSearchTester.getSearch();
        TwitterSearch tws = new TwitterSearch() {
View Full Code Here

    public void testProduceTweets() throws InterruptedException, Exception {
        final Map<Thread, Throwable> exceptionMap = new HashMap<Thread, Throwable>();
        Thread.UncaughtExceptionHandler excHandler = createExceptionMapHandler(exceptionMap);

        // fill DB with one default tag
        tagSearchTester.getSearch().bulkUpdate(Arrays.asList(new JTag("java")), tagSearchTester.getSearch().getIndexName(), true);

        ElasticUserSearch userSearch = userSearchTester.getSearch();
        ElasticTweetSearch tweetSearch = tweetSearchTester.getSearch();

        // already existing tweets must not harm
View Full Code Here

    public void testArticleContains2Sources() throws InterruptedException, Exception {
        final Map<Thread, Throwable> exceptionMap = new HashMap<Thread, Throwable>();
        Thread.UncaughtExceptionHandler excHandler = createExceptionMapHandler(exceptionMap);

        // fill DB with one default tag
        tagSearchTester.getSearch().bulkUpdate(Arrays.asList(new JTag("java")), tagSearchTester.getSearch().getIndexName(), true);

        TwitterSearch tws = new TwitterSearch() {

            @Override
            public boolean isInitialized() {
View Full Code Here

        assertEquals("peter", tag.getUser());
    }

    @Test
    public void testSave() {
        tagSearch.store(new JTag("Test"), false);
        tagSearch.store(new JTag("#Test"), false);
        tagSearch.store(new JTag("algorithm -google"), true);
        assertEquals("test", tagSearch.findByTerm("tesT").getTerm());
        assertEquals("#test", tagSearch.findByTerm("#test").getTerm());
        assertEquals("algorithm -google", tagSearch.findByTerm("algorithm -google").getTerm());

        assertEquals(3, tagSearch.findSorted(0, 1000).size());
View Full Code Here

TOP

Related Classes of de.jetwick.data.JTag

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.