/**
* Copyright (C) 2010 Peter Karich <jetwick_@_pannous_._info>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.jetwick.tw;
import com.google.inject.Inject;
import com.google.inject.Provider;
import de.jetwick.rmi.RMIClient;
import de.jetwick.data.JTweet;
import de.jetwick.data.JUser;
import de.jetwick.tw.queue.QueueThread;
import de.jetwick.util.MaxBoundSet;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import twitter4j.TwitterException;
public class MyTweetGrabber implements Serializable {
private static final long serialVersionUID = 1L;
private final Logger logger = LoggerFactory.getLogger(MyTweetGrabber.class);
private String userName;
private String queryStr;
private TwitterSearch tweetSearch;
@Inject
private Provider<RMIClient> rmiClient;
private int tweetCount;
private Collection<JTweet> tweets;
@Inject
private Provider<MaxBoundSet> lastSearches;
public MyTweetGrabber() {
}
public MyTweetGrabber init(Collection<JTweet> tweets, String query, String userName) {
this.tweets = tweets;
this.userName = userName;
this.queryStr = query;
return this;
}
public void setUserName(String userName) {
this.userName = userName.toLowerCase();
}
public String getUserName() {
return userName;
}
public MyTweetGrabber setTwitterSearch(TwitterSearch ts) {
this.tweetSearch = ts;
return this;
}
public MyTweetGrabber setRmiClient(Provider<RMIClient> rmiClient) {
this.rmiClient = rmiClient;
return this;
}
public MyTweetGrabber setTweetsCount(int count) {
tweetCount = count;
return this;
}
public QueueThread queueTweetPackage() {
return new QueueThread() {
@Override
public void run() {
if (!tweetSearch.isInitialized())
return;
int rl = tweetSearch.getRateLimit();
if (rl <= TwitterSearch.LIMIT) {
doAbort(new RuntimeException("Couldn't process query (TwitterSearch+Index)."
+ " Rate limit is smaller than " + TwitterSearch.LIMIT + ":" + rl));
return;
}
String feedSource = "";
if (tweets == null) {
if (userName != null && !userName.isEmpty()) {
// TODO exlude friendSearch
try {
if (!isSearchDoneInLastMinutes("user:" + userName)) {
// logger.info("lastsearches hashcode:" + lastSearches.hashCode());
tweets = new LinkedBlockingQueue<JTweet>();
feedSource = "grab user:" + userName;
tweets.addAll(tweetSearch.getTweets(new JUser(userName), new ArrayList<JUser>(), tweetCount));
logger.info("add " + tweets.size() + " tweets from user search: " + userName);
}
} catch (TwitterException ex) {
doAbort(ex);
logger.warn("Couldn't update user: " + userName + " " + ex.getLocalizedMessage());
}
} else if (queryStr != null && !queryStr.isEmpty()) {
try {
if (!isSearchDoneInLastMinutes(queryStr)) {
// logger.info("lastsearches hashcode:" + lastSearches.hashCode());
tweets = new LinkedBlockingQueue<JTweet>();
feedSource = "grab query:" + queryStr;
tweetSearch.search(queryStr, tweets, tweetCount, 0);
logger.info("added " + tweets.size() + " tweets via twitter search: " + queryStr);
}
} catch (TwitterException ex) {
doAbort(ex);
logger.warn("Couldn't query twitter: " + queryStr + " " + ex.getLocalizedMessage());
}
}
} else
feedSource = "filledTweets:" + tweets.size();
try {
if (tweets != null && tweets.size() > 0 && !feedSource.isEmpty()) {
for (JTweet tw : tweets) {
rmiClient.get().init().send(tw.setFeedSource(feedSource));
}
}
} catch (Exception ex) {
logger.warn("Error while sending tweets to queue server" + ex.getMessage());
}
}
};
}
public boolean isSearchDoneInLastMinutes(String string) {
return !lastSearches.get().add(string.toLowerCase());
}
public QueueThread queueArchiving() {
return new QueueThread() {
@Override
public void run() {
try {
JUser user = new JUser(userName);
int maxTweets = tweetCount;
tweetCount = 0;
int rows = 100;
setProgress(0);
logger.info("start archiving!");
for (int start = 0; start < maxTweets && !isCanceled(); start += rows) {
Collection<JTweet> tmp = tweetSearch.getTweets(user, start, rows);
if (tmp.isEmpty())
continue;
try {
tweetCount += tmp.size();
for (JTweet tw : tmp) {
tw.makePersistent();
}
for (JTweet tw : tmp) {
rmiClient.get().init().send(tw.setFeedSource("archiving user:" + userName));
}
logger.info("queue tweets " + tweetCount + " to index queue");
setProgress((int) (tweetCount * 100.0 / maxTweets));
} catch (Exception ex) {
logger.warn("Error for tweets [" + start + "," + (start + 100)
+ "] sending to index queue:", ex);
}
}
logger.info("grabbed tweets for: " + userName);
doFinish();
} catch (TwitterException ex) {
doAbort(ex);
logger.warn("Couldn't get all tweets for user: " + userName + " " + ex.getLocalizedMessage());
} catch (Exception ex) {
doAbort(ex);
logger.error("Couldn't init rmi server? " + ex.getLocalizedMessage());
}
}
};
}
public int getTweetCount() {
return tweetCount;
}
public MyTweetGrabber setMyBoundSet(final MaxBoundSet<String> boundSet) {
lastSearches = new Provider<MaxBoundSet>() {
@Override
public MaxBoundSet get() {
return boundSet;
}
};
return this;
}
}