Package de.excrawler.server

Source Code of de.excrawler.server.DownImageWorker

/*
*  Copyright (C) 2010 Yves Hoppe.  All Rights Reserved.
*  http://www.yves-hoppe.de
*
*  This is free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  This software is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  You should have received a copy of the GNU General Public License
*  along with this software; if not, write to the Free Software
*  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
*  USA.
*/

package de.excrawler.server;

import java.util.logging.*;
import java.sql.*;
import java.io.*;
import java.net.*;
import java.util.regex.Pattern;
import java.util.Date;

/**
*
* @author Yves Hoppe <info at yves-hoppe.de>
*/
public class DownImageWorker extends Thread {

    public String letter;
    public int limit;

    public static String status = "DownloadImageWorker is doing nothing";
    public static int WORKING = 1;

    Logger logger = Logger.getLogger(Main.class.getName());
    Connection connection = null;

    // Implemented later

    DownImageWorker(String chr, int lim) {
        super("downimage-" + chr);
        this.letter = chr;
        this.limit = lim;
    }

    @Override
    public void run() {       
        try {
        String currentLetter = null;
        int current = 0;

//      System.out.println("letter: " + letter);

        String pattern = "[|]";
        Pattern splitter = Pattern.compile(pattern);
        String[] result = splitter.split(letter);

        while(true) {
        String address = null;
        currentLetter = result[current];

        connection = DatabaseTools.getDBConnection();

        // Ok hardcoding it - for later seperating
        // String sql = "Select * FROM imglist_" + currentLetter + " WHERE status = 0 or status = 5 ORDER by priority ASC LIMIT 0, ?";

        String sql = "Select * FROM imglist WHERE status = 0 or status = 5 ORDER by priority ASC LIMIT 0, ?";

        PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE);
        //statement.setString(1, currentLetter);
        statement.setInt(1, limit);

        ResultSet entry = statement.executeQuery();

        int success = 0;

        while(entry.next())
        {
            address = entry.getString("url");
            status = "DownloadImageWorker downloads: " + address;
          
            try {
                URL aURL = new URL(address);

                File newfile = new File(CrawlerConfig.TMPDIR + File.separatorChar + "images" + File.separatorChar + "tmp_"+ currentLetter + "_"+ entry.getInt("id"));
                // e.g sites/a_crawlistid Todo: Move to another tmp directory;
                newfile.createNewFile();

                InputStream in = aURL.openStream();
                byte[] buffer = new byte[8192];

                FileOutputStream out = new FileOutputStream(newfile);

                int _tmp = 0;

                while((_tmp = in.read(buffer)) > 0){
                    out.write(buffer, 0 , _tmp);
                }

                in.close();
                out.close();

                success = 1;

            } catch (Exception e) {
               logger.log(Level.INFO, "Error at DownloadImageWorker " + DownImageWorker.currentThread().getName(), e); // tooo many errors out there so just info
               success = 0;
            }

            if (success == 1)
            {
               entry.updateInt("status", 1);
               entry.updateRow();
            } else {
               entry.updateInt("status", 7);
               int prior = entry.getInt("priority");
               prior += 3;
               entry.updateInt("priority", prior);
               entry.updateTimestamp("date", new java.sql.Timestamp(System.currentTimeMillis()));
               entry.updateRow();
            }
        }

        entry.close();
        statement.close();
        connection.close();

        if (WORKING == 0)
            break;

        if (current < result.length -1)
            current++;
        else
            current = 0;
        }

    } catch (Exception e) {
        logger.log(Level.SEVERE, "DownloadImageWorker " + DownImageWorker.currentThread().getName() + " hangs - restarting", e);
        DownWebWorker.currentThread().start();
    }
    } // End run

}
TOP

Related Classes of de.excrawler.server.DownImageWorker

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.