package main;
import IO.Downloader;
import callback.IDownloadCallback;
import com.google.gson.Gson;
import configuration.Configuration;
import reddit.RedditRequest;
import reddit.json.ChildData;
import reddit.json.Response;
import reddit.json.ResponseChildData;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.*;
/**
* Created by tama on 6/17/14.
*
* The program main entry point, initializes the program :
* - TODO : connect to the database
*/
public class EntryPoint {
private static void help()
{
System.out.println("Moe downloader v2");
System.out.println("------");
System.out.println("This program will download pictures liked and/or saved by an user on reddit, putting them in a dropbox folder afterwards");
System.out.println("It is meant to be launched automatically from a scheduled task (\"cron\")");
System.out.println("");
System.out.println("Parameters :");
System.out.println("\t--help\t\t\tWell... this thing.");
System.out.println("\t--conf\t\t\tThe configuration file location");
System.out.println("\t--source-url\tThe reddit source from which the links will be fetched");
System.out.println("\t--count\t\t\tThe number of items to fetch");
System.out.println("\t--reddit-token\tReddit oauth token, needed to access reddit-related features");
System.out.println("");
System.out.println("v2.0 - June 25th 2014 by tama (/u/tama_92)");
}
private static boolean parseArgs(String[] args, Configuration conf)
{
int len = args.length;
for (int i = 0; i < len - 1; ++i)
{
if (args[i].equals("--help") || args[i + 1].equals("--help")) {
help();
return false; /* boo. */
} else if (args[i].equals("--conf") || args[i].equals("-c")) {
conf.put("config_file", args[i + 1]);
} else if (args[i].equals("--source") || args[i].equals("-s")) {
conf.put("source_url", args[i + 1]);
} else if (args[i].equals("--count")) {
conf.put("count", args[i + 1]);
} else if (args[i].equals("--reddit-token")) {
conf.put("reddit_token", args[i + 1]);
}
}
return true;
}
/**
* Return the next batch of 100 links
* @return
*/
private static List<ResponseChildData> getNextLinksBatch(RedditRequest reddit, String sourceUrl, int count, String lastFetched) throws IOException {
/* API limit is 100 */
count = Math.min(count, 100);
/* Build the URL from the parameters */
String url = sourceUrl + (sourceUrl.contains("?") ? "&" : "?");
url = url + "limit=" + count;
if (lastFetched != null)
url = url + "&after=" + lastFetched;
/* Request the URL and return the response */
String liked = reddit.requestUrl(url);
Gson gson = new Gson();
Response response = gson.fromJson(liked, Response.class);
return response.getData().getChildren();
}
public static void main(String[] args)
{
final DownloadReport report = new DownloadReport();
Configuration conf = new Configuration();
if (parseArgs(args, conf))
{
try
{
/* Parse the configuration xml file */
conf.loadFile((String) conf.get("config_file"));
/* Load the plugins */
PluginLoader loader = new PluginLoader((String)(conf.get(Configuration.PLUGIN_FOLDER_TAG)));
loader.load();
/* Get the subreddits */
List<String> subreddits = (List<String>)conf.get(Configuration.SUBREDDITS_ALLOWED_TAG);
/* Get number of download threads and create the thread pool */
String downloadThreads = (String)conf.get(Configuration.PARALLEL_DOWNLOAD_TAG);
int numberOfThreads = downloadThreads == null ? 10 : Integer.valueOf(downloadThreads);
ExecutorService threadPool = Executors.newFixedThreadPool(numberOfThreads);
/* Connect to reddit and get the content from the source given */
System.out.println("Connecting to reddit");
RedditRequest reddit = new RedditRequest((String) conf.get("reddit_token"));
System.out.println("Fetching results");
String sourceUrl = (String) conf.get("source_url");
String countStr = (String) conf.get("count");
int count = countStr == null ? 25 : Integer.valueOf(countStr);
int numberOfBatchs = count / 100 + 1;
String lastFetched = null;
for (int i = 0; i < numberOfBatchs; ++i)
{
List<ResponseChildData> linkBatch = null;
try
{
linkBatch = getNextLinksBatch(reddit, sourceUrl, count, lastFetched);
}
catch (Exception e)
{
System.out.println("Failed to get next link batch.");
break;
}
System.out.println("Downloading [fetched " + linkBatch.size() + " links]");
for (ResponseChildData link : linkBatch)
{
ChildData data = link.getData();
/* Get link and title */
String url = data.getUrl();
String title = data.getTitle().replaceAll("/", " ");
String subreddit = data.getSubreddit();
if (!subreddits.contains(subreddit))
continue;
Thread t = new Thread(new Downloader(conf, loader, url, title, new IDownloadCallback() {
@Override
public void onSuccess(String title) {
report.success(title);
}
@Override
public void onError(String title, Exception e) {
report.failed(title + "(" + e.getMessage() + ")");
}
}));
threadPool.execute(t);
}
count = count - 100;
/* Get the "fullname thing" of the last link fetched */
ResponseChildData lastLink = linkBatch.get(linkBatch.size() - 1);
lastFetched = lastLink.getKind() + "_" + lastLink.getData().getId();
}
threadPool.shutdown();
//The application is run every 10 minutes, so it's useless to wait more than that
threadPool.awaitTermination(600L, TimeUnit.SECONDS);
System.out.println(report.toString());
}
catch (Exception e)
{
System.err.println("Error :(");
e.printStackTrace();
}
}
}
}