/*******************************************************************************
* HelloNzb -- The Binary Usenet Tool
* Copyright (C) 2010-2013 Matthias F. Brandstetter
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
******************************************************************************/
package me.mabra.hellonzb.nntpclient;
import me.mabra.hellonzb.HelloNzb;
import me.mabra.hellonzb.HelloNzbToolkit;
import me.mabra.hellonzb.nntpclient.nioengine.NettyNioClient;
import me.mabra.hellonzb.nntpclient.nioengine.RspHandler;
import me.mabra.hellonzb.parser.DownloadFile;
import me.mabra.hellonzb.parser.DownloadFileSegment;
import me.mabra.hellonzb.parser.SegmentQueue;
import me.mabra.hellonzb.util.MyLogger;
import sun.reflect.generics.tree.Tree;
import javax.swing.*;
import javax.swing.text.html.HTMLDocument;
import java.awt.*;
import java.io.File;
import java.lang.reflect.Array;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.*;
/**
* This class is used to download a file. It creates as many client threads as
* specified by the user settings. When all segments were completely downloaded,
* they are put together and saved to the output file.
*
* @author Matthias F. Brandstetter
*/
public class NntpFileDownloader implements Runnable
{
private static final int MAX_430_ERRORS = 100;
/** The main HelloNzb application object */
private final HelloNzb mainApp;
/** central logger object */
private MyLogger logger;
/** Nio client object */
private NettyNioClient nioClient;
/** The backup file downloader (or null if no alternative server set in preferences) */
private BackupFileDownloader backupFileDownloader;
/** The file that should has to be downloaded */
private SegmentQueue segQueue;
/** map a DownloadFile to all associated RspHandler objects */
private HashMap<DownloadFile, TreeMap<Integer, RspHandler>> dlFileRspHandlerMap;
/** A list of active response handlers */
private ArrayList<RspHandler> activeRspHandlers;
/** A list of filenames that are currently being decoded in a bg thread */
private ArrayList<FileDecoder> activeFileDecoders;
/** The download directory on local disk */
private File dlDir;
/** Flag set from main app when this thread should pause working */
private boolean pause;
/** Flag set from main app when this thread should shutdown */
private boolean shutdown;
/** CRC32 error flag */
private boolean crc32Error;
/** Count how ofter we failed to fetch an article (430) */
private int fetch430count;
/** Flag to tell whether or not the too-many-430-errors message has been sent to main app */
private boolean tooMany430ErrorsSent;
/**
* This is the constructor of the class. It has to receive the file that has
* to be downloaded by this downloader object.
*/
public NntpFileDownloader(HelloNzb mainApp, NettyNioClient nioClient, BackupFileDownloader backup,
SegmentQueue segQueue, File dlDir)
{
this.mainApp = mainApp;
this.logger = mainApp.getLogger();
this.nioClient = nioClient;
this.backupFileDownloader = backup;
this.segQueue = segQueue;
this.dlFileRspHandlerMap = new HashMap<>();
this.activeRspHandlers = new ArrayList<>();
this.activeFileDecoders = new ArrayList<>();
this.dlDir = dlDir;
this.pause = false;
this.shutdown = false;
this.crc32Error = false;
this.fetch430count = 0;
this.tooMany430ErrorsSent = false;
}
/**
* This method starts the thread and begins to download the file(s).
*/
@Override
public void run()
{
int runningThreads = 0;
final int maxThreads = Integer.parseInt(mainApp.getPrefValue("ServerSettingsThreadCount"));
HashMap<String, Integer> downloadedBytes = new HashMap<String, Integer>();
HashMap<String, Integer> lastProgBarUpdate = new HashMap<String, Integer>();
// loop at all segments of the download file
while(!shutdown && (segQueue.hasMoreSegments() || runningThreads > 0))
{
// more segments to go?
while(segQueue.hasMoreSegments() && runningThreads < maxThreads && !pause && nioClient.hasFreeSlot())
{
// get next download segment of the download file
DownloadFileSegment seg = segQueue.nextSegment();
if(seg == null)
break;
String filename = seg.getDlFile().getFilename();
logger.msg("Downloading next segment of file: " + filename, MyLogger.SEV_DEBUG);
// create new response handler
RspHandler newHandler = new RspHandler(seg, false);
synchronized(activeRspHandlers)
{
activeRspHandlers.add(newHandler);
}
// map the new response handler to the download file
mapHandlerToFile(newHandler, seg);
// start data download and increase thread counter
nioClient.fetchArticleData(seg.getGroups().firstElement(), seg.getArticleId(), newHandler);
runningThreads++;
}
// Add finished response handlers from backup downloader to this.activeRspHandlers
if(backupFileDownloader != null && backupFileDownloader.hasFinishedHandlers())
{
for(RspHandler handler : backupFileDownloader.getFinishedHandlers())
{
activeRspHandlers.add(handler);
DownloadFileSegment seg = handler.dlFileSeg();
dlFileRspHandlerMap.get(seg.getDlFile()).put(seg.getIndex(), handler);
}
}
// check if the next element of the result set is already finished
int actRspHdlsSize;
synchronized(activeRspHandlers)
{
actRspHdlsSize = activeRspHandlers.size();
}
ArrayList<RspHandler> toRemoveVector = new ArrayList<>();
for(int i = 0; i < actRspHdlsSize; i++)
{
RspHandler handler;
synchronized(activeRspHandlers)
{
handler = activeRspHandlers.get(i);
}
// handle error response from NNTP server
if(handler.getError() == RspHandler.ERR_NONE)
{
// no error, do nothing
}
else if(handler.getError() == RspHandler.ERR_AUTH)
{
// do nothing for this error (?)
}
else if(handler.getError() == RspHandler.ERR_FETCH_430)
{
// 430 No such article
if(backupFileDownloader != null && !handler.isFromBackupDownloader())
{
// try to download this failed segment via the backup connection
dlFileRspHandlerMap.get(handler.dlFileSeg().getDlFile()).remove(handler.dlFileSeg().getIndex());
backupFileDownloader.addSegment(handler.dlFileSeg());
toRemoveVector.add(handler);
continue;
}
else
{
// no backup downloader, or already tried there...
String msg = "Article not found on server (" + handler.getErrorMsg().trim() + ")";
logger.msg(msg, MyLogger.SEV_WARNING);
if(!tooMany430ErrorsSent && fetch430count++ > MAX_430_ERRORS)
{
EventQueue.invokeLater(new Runnable()
{
@Override
public void run()
{
mainApp.tooMany430Errors();
}
});
tooMany430ErrorsSent = true;
logger.msg("At least " + MAX_430_ERRORS + " articles have not been found on server(s).", MyLogger.SEV_WARNING);
}
}
}
else if(handler.getError() == RspHandler.ERR_FETCH)
{
// failed to fetch article (non-430)
String msg = "Failed to fetch article <"
+ handler.dlFileSeg().getArticleId() + "> ("
+ handler.getErrorMsg() + ")";
logger.msg(msg, MyLogger.SEV_WARNING);
}
else
{
// all other errors
String msg = "Failed to fetch article <"
+ handler.dlFileSeg().getArticleId() + "> ("
+ handler.getErrorMsg() + ")";
logger.msg(msg, MyLogger.SEV_WARNING);
shutdown = true;
}
// update downloaded byte counter ...
DownloadFile dlFile = handler.dlFileSeg().getDlFile();
String filename = dlFile.getFilename();
int bytes = 0;
Integer bytesInt = downloadedBytes.get(filename);
if(bytesInt != null)
bytes = bytesInt;
bytes += handler.newByteCount();
downloadedBytes.put(filename, bytes);
// ... and progres bar in main window
int last = 0;
Integer lastInt = lastProgBarUpdate.get(filename);
if(lastInt != null)
last = lastInt;
last = updateProgressBar(bytes, last, dlFile);
lastProgBarUpdate.put(filename, last);
// all data downloaded?
if(handler.isFinished())
{
if(handler.getError() == RspHandler.ERR_NONE)
fetch430count = 0;
toRemoveVector.add(handler);
runningThreads--;
decrSegCount(filename); // decrease main window segment counter
// segment done, so check if whole download file is finished now
dlFile.removeSegment(handler.dlFileSeg().getIndex());
if(!dlFile.hasMoreSegments())
{
try
{
handleFinishedDlFile(dlFile);
}
catch(Exception e)
{
logger.printStackTrace(e);
}
}
}
}
// remove all finished handlers, and those passed to the backup downloader, from active queue
synchronized(activeRspHandlers)
{
activeRspHandlers.removeAll(toRemoveVector);
}
toRemoveVector.clear();
// all tasks done?
if(!segQueue.hasMoreSegments() && runningThreads == 0)
{
shutdown = true;
}
try
{
Thread.sleep(10);
}
catch(InterruptedException e)
{
shutdown = true;
}
} // end of main loop
logger.msg("FileDownloader loop ended.", MyLogger.SEV_DEBUG);
}
private void mapHandlerToFile(RspHandler handler, DownloadFileSegment seg)
{
TreeMap<Integer, RspHandler> tree = dlFileRspHandlerMap.get(seg.getDlFile());
if(tree == null)
tree = new TreeMap<>();
tree.put(seg.getIndex(), handler);
dlFileRspHandlerMap.put(seg.getDlFile(), tree);
}
/**
* Check whether the given filename is currently being downloaded.
*/
public boolean isActivelyDownloaded(String filename)
{
synchronized(activeRspHandlers)
{
for(RspHandler handler : activeRspHandlers)
{
String handlerFile = handler.dlFileSeg().getDlFile().getFilename();
if(filename.equals(handlerFile))
return true;
}
}
synchronized(activeFileDecoders)
{
for(int i = 0; i < activeFileDecoders.size(); i++)
{
if(activeFileDecoders.get(i).getFilename().equals(filename))
return true;
}
}
return backupFileDownloader != null ? backupFileDownloader.isActivelyDownloaded(filename) : false;
}
/**
* Remove all those segements from the queue, that belong to the given file.
*/
public void removeFileSegmentsFromQueue(String filename)
{
segQueue.removeSegments(filename);
}
/**
* Called from main app when this thread should pause working.
*/
public void setPaused(boolean p)
{
pause = p;
}
/**
* Returns the paused state of this thread.
*
* @return The boolean value
*/
public boolean isPaused()
{
return pause;
}
/**
* Called from main app when this thread should shutdown.
*/
public void shutdown()
{
shutdown = true;
}
/**
* This method is called from the HelloYenc object when it encounters an
* crc32 error at a Yenc part. Can be ignored via application settings.
*/
public void crc32Error()
{
// disabled
/*
* String pref =
* mainApp.getPrefValue("DownloadSettingsIgnoreCrc32Error");
* if(!pref.equals("true")) crc32Error = true;
*/
}
/**
* This method is called when a whole download file has been finished
* downloading. It updates main application window and starts the decoding
* thread.
*
* @param dlFile The DownloadFile object that is finished
*/
private void handleFinishedDlFile(final DownloadFile dlFile)
{
final String filename = dlFile.getFilename();
logger.msg("File downloading finished: " + filename, MyLogger.SEV_INFO);
// notify application that download has finished
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
mainApp.fileDownloadFinished(filename);
mainApp.setProgBarToDecoding(filename, dlFile.getSegCount());
}
});
// create result vector
ArrayList<byte[]> articleData = new ArrayList<>();
for(Map.Entry<Integer, RspHandler> entry : dlFileRspHandlerMap.get(dlFile).entrySet())
{
byte[] tmpArray = removeFirstLine(entry.getValue().getData(true));
articleData.add(tmpArray);
}
// call garbage collector
TreeMap<Integer, RspHandler> tree = dlFileRspHandlerMap.remove(dlFile);
tree.clear();
tree = null;
Runtime.getRuntime().gc();
logger.msg("First line(s) dump:\n" + HelloNzbToolkit.firstLineFromByteData(articleData.get(0), 2), MyLogger.SEV_DEBUG);
// determine data encoding (yenc or UU)
String encoding = null;
boolean bHasData = false;
for(int i = 0; i < articleData.size(); i++)
{
byte[] abyteHelp = articleData.get(i);
if(abyteHelp.length > 0)
{
bHasData = true;
if(bytesEqualsString(abyteHelp, "=ybegin"))
{
encoding = "yenc";
break;
}
else if(bytesEqualsString(abyteHelp, "begin "))
{
encoding = "uu";
break;
}
}
}
if(encoding == null)
{
if(bHasData)
{
encoding = "yenc";
logger.msg("No suitable decoder (no data) found for downloaded file: " +
dlFile.getFilename() + " -- Assuming yenc.", MyLogger.SEV_WARNING);
}
else
{
// too bad, no decoder found for this file :(
logger.msg("No suitable decoder found for downloaded file (no data): "
+ dlFile.getFilename(), MyLogger.SEV_ERROR);
// update main application window
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
mainApp.fileDecodingFinished(dlFile.getFilename());
}
});
return;
}
}
// start data decoding background thread
FileDecoder fileDecoder = new FileDecoder(mainApp, this, dlDir, dlFile, articleData, encoding);
synchronized(activeFileDecoders)
{
activeFileDecoders.add(fileDecoder);
}
Thread t = new Thread(fileDecoder);
t.start();
}
/**
* Call from FileDecoder bg thread when it is done with its work.
*/
protected void fileDecoderFinished(FileDecoder fileDecoder)
{
synchronized(activeFileDecoders)
{
activeFileDecoders.remove(fileDecoder);
}
}
/**
* Update the progress bar of the currently downloaded file in main window.
* Only update if progess has at least increased by one percent of the total
* file size of the downloaded file.
*
* @param downloadedBytes The current amount of downloaded bytes
* @param lastProgBarUpdate The byte count at the last progress bar update
* @param file The download file
* @return The byte count at the last progress bar update
*/
private int updateProgressBar(int downloadedBytes, int lastProgBarUpdate,
DownloadFile file)
{
int totalSize = (int) file.getTotalFileSize();
// only update progess bar if progess has at least increased by one percent
int diff = downloadedBytes - lastProgBarUpdate;
int onePercent = (int) totalSize / 100;
if(diff >= onePercent)
{
final String filename = file.getFilename();
final int db = downloadedBytes;
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
mainApp.updateDownloadQueue(filename, db);
}
});
return downloadedBytes; // prog bar updated, so return the new byte count
}
return lastProgBarUpdate; // no update, so return the previous byte count
}
/**
* Decrease the segment count in the according row of the main window
* download table.
*
* @param filename The filename of the row to update in main window
*/
private void decrSegCount(final String filename)
{
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
mainApp.decrSegCount(filename);
}
});
}
/**
* Remove the first line from the passed byte array.
*
* @param inputArray The byte array to process
* @return The processed byte array
*/
private static byte[] removeFirstLine(byte[] inputArray)
{
int offset = 0;
while(offset < inputArray.length && (inputArray[offset] == 10 || inputArray[offset] == 13))
offset++;
while(offset < inputArray.length && inputArray[offset] != 10 && inputArray[offset] != 13)
offset++;
while(offset < inputArray.length && (inputArray[offset] == 10 || inputArray[offset] == 13))
offset++;
byte[] newArray = new byte[inputArray.length - offset];
System.arraycopy(inputArray, offset, newArray, 0, newArray.length);
return newArray;
}
/**
* Check if the first X characters of a byte stream match a String.
*
* @param data
* The byte array to process
* @param pattern
* The String to match
* @return True if the pattern was found, false otherwise
*/
private static boolean bytesEqualsString(byte[] data, String pattern)
{
byte[] bytes = new byte[pattern.length()];
Charset csets = Charset.forName("US-ASCII");
boolean fin = false;
int currChar = 0;
// remove any CR and/or LF characters at the beginning of the article
// data
while(!fin)
{
if(currChar >= data.length)
break;
byte in = data[currChar];
ByteBuffer bb = ByteBuffer.wrap(new byte[] { (byte) in });
CharBuffer cb = csets.decode(bb);
char c = cb.charAt(0);
if(data.length > 0 && (c == '\n' || c == '\r'))
currChar++;
else
fin = true;
if(data.length == 0)
fin = true;
}
// extract bytes (chars) to check from article data
for(int i = 0; i < bytes.length && i < data.length; i++, currChar++)
{
byte in = data[currChar];
bytes[i] = (byte) in;
}
// decode byte data to characters
ByteBuffer bb = ByteBuffer.wrap(bytes);
CharBuffer cb = csets.decode(bb);
// compare these characters to the pattern String
for(int i = 0; i < pattern.length(); i++)
if(cb.charAt(i) != pattern.charAt(i))
return false;
return true;
}
}