/**
* Copyright (c) 2007, Markus Jevring <markus@jevring.net>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The names of the contributors may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
package cu.ftpd.modules.zipscript.internal;
import cu.ftpd.ServiceManager;
import cu.ftpd.filesystem.filters.DashMissingFileFilter;
import cu.ftpd.filesystem.metadata.Directory;
import cu.ftpd.filesystem.metadata.Metadata;
import cu.ftpd.logging.Logging;
import cu.ftpd.user.User;
import java.io.*;
import java.text.MessageFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.zip.CRC32;
/**
* one of these are created for each race.
* they keep track of who has right to modify what files, and they also trigger sfv checks for new files
* this class owns the sfvfile for a certain race
*
* @author Markus Jevring <markus@jevring.net>
* @since 2007-maj-17 : 03:44:02
* @version $Id: Race.java 292 2009-03-04 19:44:36Z jevring $
*/
public class Race implements Serializable {
private final File racedir;
private final SfvFile sfv;
private final String section;
private String leader = "";
private String currentProgressBarDirname = "";
private boolean alreadyHalfway = false;
private boolean started;
private long estimatedSize = 0;
private long starttime = 0;
private long endtime = 0;
private long lastUpdated = 0;
private final String siteShortName;
private HashMap<String, Long> calculatedChecksums; // should we make this transient, or just load the entire race object from disk?
private final HashMap<String, Racer> racers; // username -> racerobject
private final HashMap<String, RaceGroup> groups;
private final HashMap<String, Racer> completeFiles; // filename -> username
private transient LinkedList<Racer> racerRanking;
private transient LinkedList<RaceGroup> groupRanking;
private transient RaceLog log;
private final Pattern progressbarDeletePattern = Pattern.compile("\\[.*\\] - \\[.*\\] - \\[.*\\]");
private final MessageFormat progress = new MessageFormat("[{0}] - [INCOMPLETE {1} of {2} files] - [{0}]");
// short_name, size, files
private final MessageFormat complete = new MessageFormat("[{0}] - [{1} {2} - COMPLETE ] - [{0}]");
public Race(String sfvFilePath, String section, String siteShortName) {
//long start = System.currentTimeMillis();
racers = new HashMap<String, Racer>();
groups = new HashMap<String, RaceGroup>();
calculatedChecksums = new HashMap<String, Long>();
completeFiles = new HashMap<String, Racer>();
starttime = System.currentTimeMillis();
this.section = section;
File sfvfile = new File(sfvFilePath);
racedir = sfvfile.getParentFile();
sfv = new SfvFile(sfvfile);
this.siteShortName = siteShortName;
sfv.populate(); // back to constructor. if we don't do it here, we don't know how many files are listed in the sfv
}
public synchronized void start() {
if (!started) {
sfv.populate(); // moved this from the constructor. We don't want to do heavy duty stuff in the constructor
//System.out.println("took " + (System.currentTimeMillis() - start) + " milliseconds to initialize a race");
for (Map.Entry<String, Long> file : sfv.getFiles().entrySet()) {
//System.out.println(file.getKey());
// _todo: check if passing anonymous files like this leaves us with a hojillion open file handles. use some profiler (nope,the GC should handle them)
Long precalcChecksum = calculatedChecksums.get(file.getKey());
boolean ok;
if (precalcChecksum != null) {
//System.out.println("found checksum for file " + file.getKey() + "->" + precalcChecksum);
ok = verify(new File(racedir, file.getKey()), file.getValue(), precalcChecksum);
} else {
ok = verify(new File(racedir, file.getKey()), file.getValue(), 0);
}
if (!ok) {
Racer racer = completeFiles.remove(file.getKey());
if (racer != null) {
RaceGroup group = groups.get(racer.getGroup());
RaceFile rf = racer.getFile(file.getKey());
if (rf != null) {
racer.deleteFile(rf);
group.deleteFile(rf);
saveRaceToDisk();
} // it is null when it is an .nfo or something
}
} else {
// the file is ok, and if we have it as a complete file, just let it be. if not, add it with an unknown racer.
Racer racer = completeFiles.get(file.getKey());
if (racer == null) {
// we didn't have it, then add it with an unknown racer.
// this will typically happen after a version switch, or if a directory is moved to the site via the shell
String username = "unknown";
String group = "unknown";
Directory directory = ServiceManager.getServices().getMetadataHandler().getDirectory(racedir);
if (directory != null) {
Metadata m = directory.getMetadata(file.getKey());
if (m != null) {
username = m.getUsername();
group = m.getGroupname();
}
}
completeFiles.put(file.getKey(), new Racer(username, group));
}
}
}
started = true;
createProgressBar();
}
}
public synchronized void rescan() {
calculatedChecksums = new HashMap<String, Long>();
//completeFiles = new HashMap<String, Racer>();
started = false;
start();
// .message is created above
}
/**
* Verifies the integrity of a file, based on a CRC checksum. The checksum to check against comes from the supplied sfv-file.
*
* @param file the file to be processed.
* @param checksum the checksum according to the sfv
* @param precalcChecksum this is provided from teh transfer, as the crc is being calculated on-the-fly. If not, set it to 0, and one will be calculated.
* @return true if the checksum of the file corresponded to the one in the sfv.
*/
private boolean verify(File file, long checksum, long precalcChecksum) {
boolean ok = false;
//long start = System.currentTimeMillis();
File missing = new File(file.getParentFile(), file.getName() + "-missing");
if (!file.exists()) {
ok = false;
} else {
if (precalcChecksum == 0) {
// if we didn't get a checksum from the transfer, check it normally
// since we want to be able to check multiple files in the same race at the same time, this can't be an instance variable
CRC32 checker = new CRC32();
//System.out.println("checking " + file.getPath());
byte[] buf = new byte[8192];
BufferedInputStream in = null;
try {
in = new BufferedInputStream(new FileInputStream(file));
int len;
while ((len = in.read(buf)) >= 0) {
checker.update(buf, 0, len);
}
//System.out.println("checksum took " + (System.currentTimeMillis() - start) + " milliseconds to calculate for file " + file.getName());
//System.out.println("calculated checksum: " + checker.getValue() + " provided checksum: " + checksum);
if (checker.getValue() == checksum) {
ok = true;
} else {
ok = false;
}
} catch (FileNotFoundException e) {
ok = false;
} catch (IOException e) {
// file couldn't be read, this means that we failed, and it should be removed
ok = false;
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
Logging.getErrorLog().reportException("Failed to close input stream", e);
//e.printStackTrace();
}
}
}
} else {
// we got a checksum from the transfer object
//System.out.println("file= " + file.getName() + " precalc=" + precalcChecksum + " checksum=" + checksum);
if (checksum == precalcChecksum) {
ok = true;
}
}
}
if (ok) {
calculatedChecksums.put(file.getName(), checksum);
if (missing.exists()) {
missing.delete();
}
return true;
} else {
try {
// file didn't check out, delete it.
// don't allow append when Zipscript is being used.
if(file.exists()) {
boolean delok = file.delete();
//System.err.println("Sfv check failed for: " + file.getPath() + ", deleting file..." + (delok ? " OK" : " FAILED"));
}
//System.out.println("creating missing file: " + missing.getCanonicalPath());
if (!missing.exists()) {
missing.createNewFile();
}
} catch (IOException e1) {
Logging.getErrorLog().reportError("Failed to create file: " + missing.getAbsolutePath());
}
return false;
}
}
public synchronized void createProgressBar() {
for (File file : racedir.listFiles()) {
if (file.isDirectory() && progressbarDeletePattern.matcher(file.getName()).matches()) {
file.delete();
}
}
if (isComplete()) {
currentProgressBarDirname = complete.format(new Object[]{siteShortName, cu.ftpd.logging.Formatter.size(getSize()), completeFiles.size() + "F"});
} else {
currentProgressBarDirname = progress.format(new Object[]{siteShortName, getNumerOfCurrentFiles(), getNumberOfExpectedFiles()});
}
File currentProgressBarDir = new File(racedir, currentProgressBarDirname);
//System.out.println("creating progress bar: " + currentProgressBarDirname);
currentProgressBarDir.mkdir();
}
public synchronized boolean newFile(File file, User user, long checksum, long bytesTransferred, long transferTime) {
String filename = file.getName();
Long sfvfileChecksum = sfv.getChecksum(filename);
boolean ok = false;
if (sfvfileChecksum != null) {
ok = verify(new File(racedir, filename), sfvfileChecksum, checksum);
if (ok) {
final long now = System.currentTimeMillis();
lastUpdated = now;
Racer racer = racers.get(user.getUsername());
if (racer == null) {
racer = new Racer(user.getUsername(), user.getPrimaryGroup());
racers.put(user.getUsername(), racer);
}
RaceGroup group = groups.get(user.getPrimaryGroup());
if (group == null) {
// new group
group = new RaceGroup(user.getPrimaryGroup());
groups.put(user.getPrimaryGroup(), group);
}
RaceFile rf = new RaceFile(filename, bytesTransferred, transferTime);
racer.addFile(rf);
group.addFile(rf);
completeFiles.put(filename, racer);
// we moved this here because otherwise we were announcing the racers stats before those stats were added
if (racers.size() == 1) {
// first racer
estimatedSize = sfv.getFiles().size() * bytesTransferred;
leader = user.getUsername();
log.firstRacer(this, user);
} else {
// new racer
log.newRacer(this, user);
}
if (racers.size() > 1) {
// see if there is a new leader. if there is, set it
// only do this check if we have more than one racer
String newLeader = calculateLeadingUser();
if (!leader.equals(newLeader)) {
// this means that we have a new leader
log.newLeader(this, leader, newLeader);
leader = newLeader;
}
}
if (completeFiles.size() >= Math.ceil(sfv.getFiles().size() / 2.0)) {
if (!alreadyHalfway) {
alreadyHalfway = true;
log.dirHalfway(this);
}
} else {
alreadyHalfway = false;
}
if (isComplete()) {
// the dir is complete
endtime = now;
log.raceComplete(this);
// As per request, this now displays even if there was only one racer
//if (racers.size() > 1) {
log.userStats(this);
log.groupStats(this);
//}
}
}
createProgressBar();
saveRaceToDisk();
} // otherwise it's something that's not in the sfv, like .nfo or .jpg
return ok;
// this will determine if we will add the file to dupelog or not. default to false, since we don't want to add files that were not checked against an sfv (nfo-files and jpg-files and such, which can have similar names)
// actually, files that make it in here are files to be verified anyway, so we've taken care of .nfo and .jpg and such in the calling function
}
private String calculateLeadingUser() {
int highest = 0;
String tempLeader = "";
for (Map.Entry<String, Racer> entry: racers.entrySet()) {
if (entry.getValue().getNumberOfFiles() > highest) {
highest = entry.getValue().getNumberOfFiles();
tempLeader = entry.getKey();
}
}
return tempLeader;
}
private RaceGroup calculateLeadingGroup() {
int highest = 0;
RaceGroup tempLeader = null;
for (RaceGroup group: groups.values()) {
if (group.getNumberOfFiles() > highest) {
highest = group.getNumberOfFiles();
tempLeader = group;
}
}
return tempLeader;
}
public LinkedList<Racer> getRacersInWinningOrder() {
if (racerRanking == null) {
//long start = System.currentTimeMillis();
racerRanking = new LinkedList<Racer>();
racerRanking.addAll(racers.values());
Collections.sort(racerRanking);
//System.out.println("took " + (System.currentTimeMillis() - start) + " to sort the racers' list");
}
return racerRanking;
}
public LinkedList<RaceGroup> getRaceGroupsInWinningOrder() {
if (groupRanking == null) {
//long start = System.currentTimeMillis();
groupRanking = new LinkedList<RaceGroup>();
groupRanking.addAll(this.groups.values());
Collections.sort(groupRanking);
//System.out.println("took " + (System.currentTimeMillis() - start) + " to sort the racegroups list");
}
return groupRanking;
}
public String getName() {
return racedir.getName();
}
public int getNumberOfExpectedFiles() {
return sfv.getFiles().size();
}
public int getNumerOfCurrentFiles() {
return completeFiles.size();
}
public long getEstimatedSize() {
return estimatedSize;
}
public long getSize() {
long size = 0;
File f;
for (String filename : completeFiles.keySet()) {
f = new File(racedir, filename);
size += f.length();
}
return size;
}
public long getStarttime() {
return starttime;
}
public long getEndtime() {
return endtime;
}
public long getFinalRaceSpeed() {
double dtime = (double)(endtime - starttime) / 1000.0d; // seconds, not milliseconds
return (long)(((double)estimatedSize/ 1024.0D) / dtime);
}
public long getCurrentRaceSpeed() {
double dtime = (double)(lastUpdated - starttime) / 1000.0d; // seconds, not milliseconds
return (long)(((double)estimatedSize/ 1024.0D) / dtime);
}
public String getSectionName() {
return section;
}
public String getLeader() {
return leader;
}
public int getNumberOfRacers() {
return racers.size();
}
public synchronized void deleteFile(File file, User user) {
lastUpdated = System.currentTimeMillis();
if (getNumberOfExpectedFiles() == getNumerOfCurrentFiles()) {
// the race was complete, emit the INCOMPLETE event
log.raceIncomplete(this, user);
}
Racer racer = racers.get(user.getUsername());
if (racer != null) {
RaceGroup group = groups.get(racer.getGroup());
completeFiles.remove(file.getName());
calculatedChecksums.remove(file.getName());
RaceFile rf = racer.getFile(file.getName());
if (rf != null) {
racer.deleteFile(rf);
group.deleteFile(rf);
saveRaceToDisk();
} // it is null when it is an .nfo or something
} // else the file was deleted by someone who didn't race (like an admin)
if (sfv.getChecksum(file.getName()) != null) {
// it is a race file, create the -missing
File missing = new File(file.getAbsolutePath() + "-missing");
try {
missing.createNewFile();
} catch (IOException e) {
Logging.getErrorLog().reportError("Failed to create file: " + missing.getAbsolutePath());
}
createProgressBar();
}
if (file.getName().endsWith(".sfv")) {
// delete all -missing files
File[] missingFiles = racedir.listFiles(new DashMissingFileFilter());
for (File missingFile : missingFiles) {
missingFile.delete();
}
// delete the .raceinfo
File raceinfo = new File(racedir, ".raceinfo");
boolean ok = raceinfo.delete();
//System.out.println("deleting .raceinfo in: (" + raceinfo.getParentFile().getAbsolutePath() + ") and it was " + (ok ? "successful":"not successful"));
// delete the progress bar
File bar = new File(racedir, currentProgressBarDirname);
ok = bar.delete();
//System.out.println("deleting progress bar: " + currentProgressBarDirname + "(" + bar.getAbsolutePath() + ") and it was " + (ok ? "successful":"not successful"));
}
}
public boolean isComplete() {
//System.out.println("isComplete: sfv: " + sfv.getFiles().size() + " complete: " + completeFiles.size());
return sfv.getFiles().size() == completeFiles.size(); // this works, since it is only files that exists in the sfv that get added to the set of complete files
//return sfv.getFiles().keySet().equals(completeFiles.keySet()); // _todo: this would be faster (but possibly less accurate) if we used .size() instead.
}
private void saveRaceToDisk() {
ObjectOutputStream oos = null;
try {
//long start = System.currentTimeMillis();
oos = new ObjectOutputStream(new FileOutputStream(new File(racedir,".raceinfo")));
oos.writeObject(this);
//System.out.println("took " + (System.currentTimeMillis() - start) + " milliseconds to serialize and store a race object");
} catch (IOException e) {
Logging.getErrorLog().reportError("Writing race to disk failed for file: " + new File(racedir,".raceinfo").getAbsolutePath());
} finally {
if (oos != null){
try {
oos.close();
} catch (IOException e) {
Logging.getErrorLog().reportError("Closing race file failed: " + new File(racedir,".raceinfo").getAbsolutePath());
}
}
}
}
public HashMap<String, Racer> getRacers() {
return racers;
}
public HashMap<String, RaceGroup> getGroups() {
return groups;
}
public Racer getRacer(String username) {
return racers.get(username);
}
public RaceGroup getLeadingGroup() {
return calculateLeadingGroup();
}
public Racer getRacerOfFile(String filename) {
return completeFiles.get(filename);
}
public File getRacedir() {
return racedir;
}
public long getLastUpdated() {
return lastUpdated;
}
/**
* We have to set the log here, because we serialize it, and the log isn't a serializable object by nature.
* @param log the logger to use when logging events.
*/
public void setLog(RaceLog log) {
this.log = log;
}
}