System.err.println();
Options op = new Options(ctpp);
if (argMap.containsKey("-stats")) {
String[] statArgs = (argMap.get("-stats"));
MemoryTreebank rawTrainTreebank = op.tlpParams.memoryTreebank();
FileFilter trainFilt = new NumberRangesFileFilter(statArgs[1], false);
rawTrainTreebank.loadPath(new File(statArgs[0]), trainFilt);
System.err.println("Done reading trees.");
MemoryTreebank trainTreebank;
if (argMap.containsKey("-annotate")) {
trainTreebank = new MemoryTreebank();
TreeAnnotator annotator = new TreeAnnotator(ctpp.headFinder(), ctpp, op);
for (Tree tree : rawTrainTreebank) {
trainTreebank.add(annotator.transformTree(tree));
}
System.err.println("Done annotating trees.");
} else {
trainTreebank = rawTrainTreebank;
}
printStats(trainTreebank, pw);
System.exit(0);
}
int maxLength = 1000000;
// Test.verbose = true;
if (argMap.containsKey("-norm")) {
op.testOptions.lengthNormalization = true;
}
if (argMap.containsKey("-maxLength")) {
maxLength = Integer.parseInt((argMap.get("-maxLength"))[0]);
}
op.testOptions.maxLength = 120;
boolean combo = argMap.containsKey("-combo");
if (combo) {
ctpp.useCharacterBasedLexicon = true;
op.testOptions.maxSpanForTags = 10;
op.doDep = false;
op.dcTags = false;
}
LexicalizedParser lp = null;
Lexicon lex = null;
if (argMap.containsKey("-parser")) {
String[] parserArgs = (argMap.get("-parser"));
if (parserArgs.length > 1) {
FileFilter trainFilt = new NumberRangesFileFilter(parserArgs[1], false);
lp = LexicalizedParser.trainFromTreebank(parserArgs[0], trainFilt, op);
if (parserArgs.length == 3) {
String filename = parserArgs[2];
System.err.println("Writing parser in serialized format to file " + filename + " ");
System.err.flush();
ObjectOutputStream out = IOUtils.writeStreamFromString(filename);
out.writeObject(lp);
out.close();
System.err.println("done.");
}
} else {
String parserFile = parserArgs[0];
lp = LexicalizedParser.loadModel(parserFile, op);
}
lex = lp.getLexicon();
op = lp.getOp();
ctpp = (ChineseTreebankParserParams) op.tlpParams;
}
if (argMap.containsKey("-rad")) {
ctpp.useUnknownCharacterModel = true;
}
if (argMap.containsKey("-lengthPenalty")) {
ctpp.lengthPenalty = Double.parseDouble((argMap.get("-lengthPenalty"))[0]);
}
if (argMap.containsKey("-penaltyType")) {
ctpp.penaltyType = Integer.parseInt((argMap.get("-penaltyType"))[0]);
}
if (argMap.containsKey("-lex")) {
String[] lexArgs = (argMap.get("-lex"));
if (lexArgs.length > 1) {
Index<String> wordIndex = new HashIndex<String>();
Index<String> tagIndex = new HashIndex<String>();
lex = ctpp.lex(op, wordIndex, tagIndex);
MemoryTreebank rawTrainTreebank = op.tlpParams.memoryTreebank();
FileFilter trainFilt = new NumberRangesFileFilter(lexArgs[1], false);
rawTrainTreebank.loadPath(new File(lexArgs[0]), trainFilt);
System.err.println("Done reading trees.");
MemoryTreebank trainTreebank;
if (argMap.containsKey("-annotate")) {
trainTreebank = new MemoryTreebank();
TreeAnnotator annotator = new TreeAnnotator(ctpp.headFinder(), ctpp, op);
for (Iterator iter = rawTrainTreebank.iterator(); iter.hasNext();) {
Tree tree = (Tree) iter.next();
tree = annotator.transformTree(tree);
trainTreebank.add(tree);
}
System.err.println("Done annotating trees.");
} else {
trainTreebank = rawTrainTreebank;
}
lex.initializeTraining(trainTreebank.size());
lex.train(trainTreebank);
lex.finishTraining();
System.err.println("Done training lexicon.");
if (lexArgs.length == 3) {
String filename = lexArgs.length == 3 ? lexArgs[2] : "parsers/chineseCharLex.ser.gz";
System.err.println("Writing lexicon in serialized format to file " + filename + " ");
System.err.flush();
ObjectOutputStream out = IOUtils.writeStreamFromString(filename);
out.writeObject(lex);
out.close();
System.err.println("done.");
}
} else {
String lexFile = lexArgs.length == 1 ? lexArgs[0] : "parsers/chineseCharLex.ser.gz";
System.err.println("Reading Lexicon from file " + lexFile);
ObjectInputStream in = IOUtils.readStreamFromString(lexFile);
try {
lex = (Lexicon) in.readObject();
} catch (ClassNotFoundException e) {
throw new RuntimeException("Bad serialized file: " + lexFile);
}
in.close();
}
}
if (argMap.containsKey("-test")) {
boolean segmentWords = ctpp.segment;
boolean parse = lp != null;
assert (parse || segmentWords);
// WordCatConstituent.collinizeWords = argMap.containsKey("-collinizeWords");
// WordCatConstituent.collinizeTags = argMap.containsKey("-collinizeTags");
WordSegmenter seg = null;
if (segmentWords) {
seg = (WordSegmenter) lex;
}
String[] testArgs = (argMap.get("-test"));
MemoryTreebank testTreebank = op.tlpParams.memoryTreebank();
FileFilter testFilt = new NumberRangesFileFilter(testArgs[1], false);
testTreebank.loadPath(new File(testArgs[0]), testFilt);
TreeTransformer subcategoryStripper = op.tlpParams.subcategoryStripper();
TreeTransformer collinizer = ctpp.collinizer();
WordCatEquivalenceClasser eqclass = new WordCatEquivalenceClasser();
WordCatEqualityChecker eqcheck = new WordCatEqualityChecker();