/*
* This file is part of Skript.
*
* Skript is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Skript is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Skript. If not, see <http://www.gnu.org/licenses/>.
*
*
* Copyright 2011-2014 Peter Güttinger
*
*/
package ch.njol.skript.variables;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.jdt.annotation.Nullable;
import ch.njol.skript.Skript;
import ch.njol.skript.config.SectionNode;
import ch.njol.skript.lang.Variable;
import ch.njol.skript.log.SkriptLogger;
import ch.njol.skript.registrations.Classes;
import ch.njol.skript.util.ExceptionUtils;
import ch.njol.skript.util.FileUtils;
import ch.njol.skript.util.Task;
import ch.njol.skript.util.Utils;
import ch.njol.skript.util.Version;
/**
* TODO use a database (SQLite) instead and only load a limited amount of variables into RAM - e.g. 2 GB (configurable). If more variables are available they will be loaded when
* accessed. (rem: print a warning when Skript starts)
* rem: store null variables to prevent looking up the same variables over and over again
*
* @author Peter Güttinger
*/
public class FlatFileStorage extends VariablesStorage {
@SuppressWarnings("null")
public final static Charset UTF_8 = Charset.forName("UTF-8");
@Nullable
private volatile PrintWriter changesWriter;
private volatile boolean loaded = false;
final AtomicInteger changes = new AtomicInteger(0);
private final int REQUIRED_CHANGES_FOR_RESAVE = 1000;
@Nullable
private Task saveTask;
private boolean loadError = false;
protected FlatFileStorage(final String name) {
super(name);
}
/**
* Doesn'ts lock the connection as required by {@link Variables#variableLoaded(String, Object, VariablesStorage)}.
*/
@SuppressWarnings({"deprecation"})
@Override
protected boolean load_i(final SectionNode n) {
SkriptLogger.setNode(null);
IOException ioEx = null;
int unsuccessful = 0;
final StringBuilder invalid = new StringBuilder();
Version varVersion = Skript.getVersion(); // will be set later
final Version v2_0_beta3 = new Version(2, 0, "beta 3");
boolean update2_0_beta3 = false;
final Version v2_1 = new Version(2, 1);
boolean update2_1 = false;
BufferedReader r = null;
try {
r = new BufferedReader(new InputStreamReader(new FileInputStream(file), UTF_8));
String line = null;
int lineNum = 0;
while ((line = r.readLine()) != null) {
lineNum++;
line = line.trim();
if (line.isEmpty() || line.startsWith("#")) {
if (line.startsWith("# version:")) {
try {
varVersion = new Version("" + line.substring("# version:".length()).trim());
update2_0_beta3 = varVersion.isSmallerThan(v2_0_beta3);
update2_1 = varVersion.isSmallerThan(v2_1);
} catch (final IllegalArgumentException e) {}
}
continue;
}
final String[] split = splitCSV(line);
if (split == null || split.length != 3) {
Skript.error("invalid amount of commas in line " + lineNum + " ('" + line + "')");
if (invalid.length() != 0)
invalid.append(", ");
invalid.append(split == null ? "<unknown>" : split[0]);
unsuccessful++;
continue;
}
if (split[1].equals("null")) {
Variables.variableLoaded("" + split[0], null, this);
} else {
Object d;
if (update2_1)
d = Classes.deserialize("" + split[1], "" + split[2]);
else
d = Classes.deserialize("" + split[1], decode("" + split[2]));
if (d == null) {
if (invalid.length() != 0)
invalid.append(", ");
invalid.append(split[0]);
unsuccessful++;
continue;
}
if (d instanceof String && update2_0_beta3) {
d = Utils.replaceChatStyles((String) d);
}
Variables.variableLoaded("" + split[0], d, this);
}
}
} catch (final IOException e) {
loadError = true;
ioEx = e;
} finally {
if (r != null) {
try {
r.close();
} catch (final IOException e) {}
}
}
final File file = this.file;
if (file == null) {
assert false : this;
return false;
}
if (ioEx != null || unsuccessful > 0 || update2_1) {
if (unsuccessful > 0) {
Skript.error(unsuccessful + " variable" + (unsuccessful == 1 ? "" : "s") + " could not be loaded!");
Skript.error("Affected variables: " + invalid.toString());
}
if (ioEx != null) {
Skript.error("An I/O error occurred while loading the variables: " + ExceptionUtils.toString(ioEx));
Skript.error("This means that some to all variables could not be loaded!");
}
try {
if (update2_1) {
Skript.info("[2.1] updating " + file.getName() + " to the new format...");
}
final File bu = FileUtils.backup(file);
Skript.info("Created a backup of " + file.getName() + " as " + bu.getName());
loadError = false;
} catch (final IOException ex) {
Skript.error("Could not backup " + file.getName() + ": " + ex.getMessage());
}
}
if (update2_1) {
saveVariables(false);
Skript.info(file.getName() + " successfully updated.");
}
synchronized (fileLock) { // only synchronised because of the assertion in connect()
connect();
}
saveTask = new Task(Skript.getInstance(), 5 * 60 * 20, 5 * 60 * 20, true) {
@Override
public void run() {
if (changes.get() >= REQUIRED_CHANGES_FOR_RESAVE) {
try {
Variables.getReadLock().lock();
saveVariables(false);
changes.set(0);
} finally {
Variables.getReadLock().unlock();
}
}
}
};
return ioEx == null;
}
@Override
protected boolean requiresFile() {
return true;
}
@Override
protected File getFile(final String file) {
return new File(file);
}
final static String encode(final byte[] data) {
final char[] r = new char[data.length * 2];
for (int i = 0; i < data.length; i++) {
r[2 * i] = Character.toUpperCase(Character.forDigit((data[i] & 0xF0) >>> 4, 16));
r[2 * i + 1] = Character.toUpperCase(Character.forDigit(data[i] & 0xF, 16));
}
return new String(r);
}
final static byte[] decode(final String hex) {
final byte[] r = new byte[hex.length() / 2];
for (int i = 0; i < r.length; i++) {
r[i] = (byte) ((Character.digit(hex.charAt(2 * i), 16) << 4) + Character.digit(hex.charAt(2 * i + 1), 16));
}
return r;
}
@SuppressWarnings("null")
private final static Pattern csv = Pattern.compile("(?<=^|,)\\s*([^\",]*|\"([^\"]|\"\")*\")\\s*(,|$)");
@Nullable
final static String[] splitCSV(final String line) {
final Matcher m = csv.matcher(line);
int lastEnd = 0;
final ArrayList<String> r = new ArrayList<String>();
while (m.find()) {
if (lastEnd != m.start())
return null;
final String v = m.group(1);
if (v.startsWith("\""))
r.add(v.substring(1, v.length() - 1).replace("\"\"", "\""));
else
r.add(v.trim());
lastEnd = m.end();
}
if (lastEnd != line.length())
return null;
return r.toArray(new String[r.size()]);
}
@Override
protected boolean save(final String name, final @Nullable String type, final @Nullable byte[] value) {
synchronized (fileLock) {
if (!loaded) {
assert type == null;
return true; // deleting variables is not really required for this kind of storage, as it will be completely rewritten every once in a while, and at least once when the server stops.
}
PrintWriter cw;
while ((cw = changesWriter) == null) {
try {
fileLock.wait();
} catch (final InterruptedException e) {}
}
writeCSV(cw, name, type, value == null ? "" : encode(value));
cw.flush();
changes.incrementAndGet();
}
return true;
}
/**
* Use with find()
*/
@SuppressWarnings("null")
private final static Pattern containsWhitespace = Pattern.compile("\\s");
private final static void writeCSV(final PrintWriter pw, final String... values) {
assert values.length == 3; // name, type, value
for (int i = 0; i < values.length; i++) {
if (i != 0)
pw.print(", ");
String v = values[i];
if (v != null && (v.contains(",") || v.contains("\"") || v.contains("#") || containsWhitespace.matcher(v).find()))
v = '"' + v.replace("\"", "\"\"") + '"';
pw.print(v);
}
pw.println();
}
@Override
protected final void disconnect() {
synchronized (fileLock) {
clearChangesQueue();
final PrintWriter cw = changesWriter;
if (cw != null) {
cw.close();
changesWriter = null;
}
}
}
@Override
protected final boolean connect() {
synchronized (fileLock) {
if (changesWriter != null)
return true;
try {
changesWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file, true), UTF_8));
loaded = true;
return true;
} catch (final FileNotFoundException e) {
Skript.exception(e);
return false;
}
}
}
@Override
public void close() {
clearChangesQueue();
super.close();
saveVariables(true); // also closes the writer
}
/**
* Completely rewrites the while file
*
* @param finalSave whether this is the last save in this session or not.
*/
public final void saveVariables(final boolean finalSave) {
if (finalSave) {
final Task st = saveTask;
if (st != null)
st.cancel();
final Task bt = backupTask;
if (bt != null)
bt.cancel();
}
try {
Variables.getReadLock().lock();
synchronized (fileLock) {
try {
final File f = file;
if (f == null) {
assert false : this;
return;
}
disconnect();
if (loadError) {
try {
final File backup = FileUtils.backup(f);
Skript.info("Created a backup of your old variables.csv as " + backup.getName());
loadError = false;
} catch (final IOException e) {
Skript.error("Could not backup the old variables.csv: " + ExceptionUtils.toString(e));
Skript.error("No variables are saved!");
return;
}
}
final File tempFile = new File(Skript.getInstance().getDataFolder(), "variables.csv.temp");
PrintWriter pw = null;
try {
pw = new PrintWriter(tempFile, "UTF-8");
pw.println("# === Skript's variable storage ===");
pw.println("# Please do not modify this file manually!");
pw.println("#");
pw.println("# version: " + Skript.getVersion());
pw.println();
save(pw, "", Variables.getVariables());
pw.println();
pw.flush();
pw.close();
FileUtils.move(tempFile, f, true);
} catch (final IOException e) {
Skript.error("Unable to make a final save of the database '" + databaseName + "' (no variables are lost): " + ExceptionUtils.toString(e)); // FIXME happens at random - check locks/threads
} finally {
if (pw != null)
pw.close();
}
} finally {
if (!finalSave) {
connect();
fileLock.notifyAll();
}
}
}
} finally {
Variables.getReadLock().unlock();
}
}
/**
* Saves the variables.
* <p>
* This method uses the sorted variables map to save the variables in order.
*
* @param pw
* @param parent The parent's name with {@link Variable#SEPARATOR} at the end
* @param map
*/
@SuppressWarnings("unchecked")
private final void save(final PrintWriter pw, final String parent, final TreeMap<String, Object> map) {
outer: for (final Entry<String, Object> e : map.entrySet()) {
final Object val = e.getValue();
if (val == null)
continue;
if (val instanceof TreeMap) {
save(pw, parent + e.getKey() + Variable.SEPARATOR, (TreeMap<String, Object>) val);
} else {
final String name = (e.getKey() == null ? parent.substring(0, parent.length() - Variable.SEPARATOR.length()) : parent + e.getKey());
for (final VariablesStorage s : Variables.storages) {
if (s != this && s.accept(name))
continue outer;
}
final SerializedVariable.Value value = Classes.serialize(val);
if (value != null)
writeCSV(pw, name, value.type, encode(value.data));
}
}
}
}