/*
* Copyright (C) 2010 Yves Hoppe. All Rights Reserved.
* http://www.yves-hoppe.de
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
* USA.
*/
package de.excrawler.server;
import java.util.logging.*;
import java.sql.*;
import java.io.*;
import java.net.*;
import java.util.regex.Pattern;
import java.util.Date;
/**
*
* @author Yves Hoppe <info at yves-hoppe.de>
*/
public class DownImageWorker extends Thread {
public String letter;
public int limit;
public static String status = "DownloadImageWorker is doing nothing";
public static int WORKING = 1;
Logger logger = Logger.getLogger(Main.class.getName());
Connection connection = null;
// Implemented later
DownImageWorker(String chr, int lim) {
super("downimage-" + chr);
this.letter = chr;
this.limit = lim;
}
@Override
public void run() {
try {
String currentLetter = null;
int current = 0;
// System.out.println("letter: " + letter);
String pattern = "[|]";
Pattern splitter = Pattern.compile(pattern);
String[] result = splitter.split(letter);
while(true) {
String address = null;
currentLetter = result[current];
connection = DatabaseTools.getDBConnection();
// Ok hardcoding it - for later seperating
// String sql = "Select * FROM imglist_" + currentLetter + " WHERE status = 0 or status = 5 ORDER by priority ASC LIMIT 0, ?";
String sql = "Select * FROM imglist WHERE status = 0 or status = 5 ORDER by priority ASC LIMIT 0, ?";
PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE);
//statement.setString(1, currentLetter);
statement.setInt(1, limit);
ResultSet entry = statement.executeQuery();
int success = 0;
while(entry.next())
{
address = entry.getString("url");
status = "DownloadImageWorker downloads: " + address;
try {
URL aURL = new URL(address);
File newfile = new File(CrawlerConfig.TMPDIR + File.separatorChar + "images" + File.separatorChar + "tmp_"+ currentLetter + "_"+ entry.getInt("id"));
// e.g sites/a_crawlistid Todo: Move to another tmp directory;
newfile.createNewFile();
InputStream in = aURL.openStream();
byte[] buffer = new byte[8192];
FileOutputStream out = new FileOutputStream(newfile);
int _tmp = 0;
while((_tmp = in.read(buffer)) > 0){
out.write(buffer, 0 , _tmp);
}
in.close();
out.close();
success = 1;
} catch (Exception e) {
logger.log(Level.INFO, "Error at DownloadImageWorker " + DownImageWorker.currentThread().getName(), e); // tooo many errors out there so just info
success = 0;
}
if (success == 1)
{
entry.updateInt("status", 1);
entry.updateRow();
} else {
entry.updateInt("status", 7);
int prior = entry.getInt("priority");
prior += 3;
entry.updateInt("priority", prior);
entry.updateTimestamp("date", new java.sql.Timestamp(System.currentTimeMillis()));
entry.updateRow();
}
}
entry.close();
statement.close();
connection.close();
if (WORKING == 0)
break;
if (current < result.length -1)
current++;
else
current = 0;
}
} catch (Exception e) {
logger.log(Level.SEVERE, "DownloadImageWorker " + DownImageWorker.currentThread().getName() + " hangs - restarting", e);
DownWebWorker.currentThread().start();
}
} // End run
}