package edu.stanford.nlp.wordseg;
import java.io.*;
import java.util.*;
import java.util.regex.Pattern;
import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.io.EncodingPrintWriter;
import edu.stanford.nlp.process.ChineseDocumentToSentenceProcessor;
import edu.stanford.nlp.trees.international.pennchinese.ChineseUtils;
import edu.stanford.nlp.util.StringUtils;
import java.util.zip.GZIPInputStream;
import org.apache.hadoop.fs.FSDataInputStream;
/** This class provides a main method that loads various dictionaries, and
* saves them in a serialized version, and runtime compiles them into a word list used as a feature in the segmenter, and
* @author Pi-Chuan Chang
*/
public class ChineseDictionary {
private static final boolean DEBUG = false;
public static final int MAX_LEXICON_LENGTH = 6;
@SuppressWarnings({"unchecked"})
Set<String>[] words_ = new HashSet[MAX_LEXICON_LENGTH+1];
private ChineseDocumentToSentenceProcessor cdtos_; // = null;
private void serializeDictionary(String serializePath) {
System.err.print("Serializing dictionaries to " + serializePath + "...");
try {
ObjectOutputStream oos = IOUtils.writeStreamFromString(serializePath);
//oos.writeObject(MAX_LEXICON_LENGTH);
oos.writeObject(words_);
//oos.writeObject(cdtos_);
oos.close();
System.err.println("done.");
} catch (Exception e) {
System.err.println("Failed");
throw new RuntimeException(e);
}
}
@SuppressWarnings({"unchecked"})
private static Set<String>[] loadDictionary(String serializePath) {
Set<String>[] dict = new HashSet[MAX_LEXICON_LENGTH+1];
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
dict[i] = new HashSet<String>();
}
System.err.print("loading dictionaries from " + serializePath + "...");
try {
BufferedInputStream bis = new BufferedInputStream(
new GZIPInputStream(new FileInputStream(new File(serializePath))));
ObjectInputStream ois = new ObjectInputStream(bis);
//ChineseDictionary.MAX_LEXICON_LENGTH = (int) ois.readObject();
dict = (Set<String>[]) ois.readObject();
//cdtos_ = (ChineseDocumentToSentenceProcessor) ois.readObject();
bis.close();
} catch (Exception e) {
System.err.println("Failed");
throw new RuntimeException(e);
}
return dict;
}
@SuppressWarnings({"unchecked"})
private static Set<String>[] loadDictionary(FSDataInputStream stream) {
Set<String>[] dict = new HashSet[MAX_LEXICON_LENGTH+1];
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
dict[i] = new HashSet<String>();
}
System.err.print("loading dictionaries from " + stream.toString() + "...");
try {
// BufferedInputStream bis = new BufferedInputStream(
// new GZIPInputStream(stream));
ObjectInputStream ois = new ObjectInputStream(stream);
//ChineseDictionary.MAX_LEXICON_LENGTH = (int) ois.readObject();
dict = (Set<String>[]) ois.readObject();
//cdtos_ = (ChineseDocumentToSentenceProcessor) ois.readObject();
// bis.close();
} catch (Exception e) {
System.err.println("Failed");
throw new RuntimeException(e);
}
return dict;
}
public ChineseDictionary(String dict) {
this(new String[] { dict });
}
public ChineseDictionary(String[] dicts) {
this(dicts, null);
}
public ChineseDictionary(String[] dicts,
ChineseDocumentToSentenceProcessor cdtos) {
this(dicts, cdtos, false);
}
/**
* The first argument can be one file path, or multiple files separated by
* commas
*
**/
public ChineseDictionary(String serDicts,
ChineseDocumentToSentenceProcessor cdtos,
boolean expandMidDot) {
this(serDicts.split(","), cdtos, expandMidDot);
}
public ChineseDictionary(String[] dicts,
ChineseDocumentToSentenceProcessor cdtos,
boolean expandMidDot) {
System.err.println("Loading Chinese dictionaries from "
+dicts.length
+" files:");
for(String dict : dicts) {
System.err.println(" "+dict);
}
System.err.println();
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
words_[i] = new HashSet<String>();
}
this.cdtos_ = cdtos;
for(String dict : dicts) {
if(dict.endsWith("ser.gz")) {
Set<String>[] dictwords = loadDictionary(dict);
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
words_[i].addAll(dictwords[i]);
dictwords[i] = null;
}
} else {
addDict(dict, expandMidDot);
}
}
int total = 0;
for(int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
total += words_[i].size();
}
System.err.println("Done. Unique words in ChineseDictionary is: " + total);
}
/**
* @author ferhanture
*
* @assume dictionary needs to be in single file and gzipped
**/
public ChineseDictionary(FSDataInputStream stream,
ChineseDocumentToSentenceProcessor cdtos,
boolean expandMidDot) {
System.err.println("Loading Chinese dictionaries from "
+"1 HDFS file:"+stream.toString());
System.err.println();
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
words_[i] = new HashSet<String>();
}
this.cdtos_ = cdtos;
Set<String>[] dictwords = loadDictionary(stream);
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
words_[i].addAll(dictwords[i]);
dictwords[i] = null;
}
int total = 0;
for(int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
total += words_[i].size();
}
System.err.println("Done. Unique words in ChineseDictionary is: " + total);
}
private final Pattern midDot = Pattern.compile(ChineseUtils.MID_DOT_REGEX_STR);
private void addDict(String dict, boolean expandMidDot) {
String content = IOUtils.slurpFileNoExceptions(dict,"utf-8");
String[] lines = content.split("\n");
System.err.println(" " + dict + ": " + lines.length + " entries");
for (String line : lines) {
line = line.trim();
// normalize any midDot
if (expandMidDot) {
// normalize down middot chars
line = line.replaceAll(ChineseUtils.MID_DOT_REGEX_STR, "\u00B7");
}
addOneDict(line);
if (DEBUG) EncodingPrintWriter.err.println("ORIG: " + line, "UTF-8");
if (expandMidDot && midDot.matcher(line).find()) {
line = line.replaceAll(ChineseUtils.MID_DOT_REGEX_STR, "");
if (DEBUG) EncodingPrintWriter.err.println("ALSO: " + line, "UTF-8");
addOneDict(line);
}
}
}
private void addDict(FSDataInputStream stream, boolean expandMidDot) {
String content = IOUtils.slurpFileNoExceptions(stream,"utf-8");
String[] lines = content.split("\n");
System.err.println(" " + stream.toString() + ": " + lines.length + " entries");
for (String line : lines) {
line = line.trim();
// normalize any midDot
if (expandMidDot) {
// normalize down middot chars
line = line.replaceAll(ChineseUtils.MID_DOT_REGEX_STR, "\u00B7");
}
addOneDict(line);
if (DEBUG) EncodingPrintWriter.err.println("ORIG: " + line, "UTF-8");
if (expandMidDot && midDot.matcher(line).find()) {
line = line.replaceAll(ChineseUtils.MID_DOT_REGEX_STR, "");
if (DEBUG) EncodingPrintWriter.err.println("ALSO: " + line, "UTF-8");
addOneDict(line);
}
}
}
private void addOneDict(String item) {
int length = item.length();
if (length == 0) {
// Do nothing for empty items
} else if (length <= MAX_LEXICON_LENGTH-1) {
if (cdtos_ != null) {
item = cdtos_.normalization(item);
}
if (DEBUG) EncodingPrintWriter.err.println("DICT: "+item, "UTF-8");
words_[length].add(item);
} else {
// insist on new String as it may save memory
String subitem = new String(item.substring(0,MAX_LEXICON_LENGTH));
if (cdtos_ != null) {
subitem = cdtos_.normalization(subitem);
}
if (DEBUG) EncodingPrintWriter.err.println("DICT: "+subitem, "UTF-8");
// length=MAX_LEXICON_LENGTH and MAX_LEXICON_LENGTH+
words_[MAX_LEXICON_LENGTH].add(subitem);
}
}
public boolean contains(String word) {
int length = word.length();
if (length <= MAX_LEXICON_LENGTH-1) {
return words_[length].contains(word);
} else {
length = MAX_LEXICON_LENGTH;
return words_[length].contains(word.substring(0,6));
}
}
public static void main(String[] args) {
String inputDicts = "/u/nlp/data/chinese-dictionaries/plain/ne_wikipedia-utf8.txt,/u/nlp/data/chinese-dictionaries/plain/newsexplorer_entities_utf8.txt,/u/nlp/data/chinese-dictionaries/plain/Ch-name-list-utf8.txt,/u/nlp/data/chinese-dictionaries/plain/wikilex-20070908-zh-en.txt,/u/nlp/data/chinese-dictionaries/plain/adso-1.25-050405-monolingual-clean.utf8.txt,/u/nlp/data/chinese-dictionaries/plain/lexicon_108k_normalized.txt,/u/nlp/data/chinese-dictionaries/plain/lexicon_mandarintools_normalized.txt,/u/nlp/data/chinese-dictionaries/plain/harbin-ChineseNames_utf8.txt,/u/nlp/data/chinese-dictionaries/plain/lexicon_HowNet_normalized.txt";
String output = "/u/nlp/data/gale/segtool/stanford-seg/classifiers/dict-chris6.ser.gz";
Map<String,Integer> flagMap = new HashMap<String,Integer>();
flagMap.put("-inputDicts", 1);
flagMap.put("-output", 1);
Map<String,String[]> argsMap = StringUtils.argsToMap(args,flagMap);
// args = argsMap.get(null);
if(argsMap.keySet().contains("-inputDicts")) {
inputDicts = argsMap.get("-inputDicts")[0];
}
if(argsMap.keySet().contains("-output")) {
output = argsMap.get("-output")[0];
}
String[] dicts = inputDicts.split(",");
ChineseDocumentToSentenceProcessor cdtos
= new ChineseDocumentToSentenceProcessor(null);
boolean expandMidDot = true;
ChineseDictionary dict = new ChineseDictionary(dicts, cdtos, expandMidDot);
dict.serializeDictionary(output);
/*
//ChineseDictionary dict = new ChineseDictionary(args[0]);
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
System.err.println("Length: " + i+": "+dict.words[i].size());
}
for (int i = 0; i <= MAX_LEXICON_LENGTH; i++) {
System.err.println("Length: " + i+": "+dict.words[i].size());
if (dict.words[i].size() < 1000) {
for (String word : dict.words[i]) {
EncodingPrintWriter.err.println(word, "UTF-8");
}
}
}
for (int i = 1; i < args.length; i++) {
System.err.println(args[i] + " " + Boolean.valueOf(dict.contains(args[i])).toString());
}
*/
}
}