String VOCABDIR = "data/vocab"; // /Users/ferhanture/Documents/workspace/ivory-github/Ivory/data/vocab
String TOKENDIR = "data/tokenizer"; // "/Users/ferhanture/Documents/workspace/ivory-github/Ivory/data/tokenizer
String DATADIR = "/fs/clip-qa/ferhan/cl-pwsim/pwsim-experiments-2013"; // /Users/ferhanture/edu/research_archive/data/de-en/eu-nc-wmt08
BitextClassifierUtils dt = new BitextClassifierUtils();
numSentencesPerDocE = new HMapSIW();
numSentencesPerDocF = new HMapSIW();
FileSystem localFs = FileSystem.getLocal(new Configuration());
eVocabSrc = HadoopAlign.loadVocab(new Path(VOCABDIR+"/vocab.en-de.en"), localFs);
eVocabTrg = HadoopAlign.loadVocab(new Path(VOCABDIR+"/vocab.de-en.en"), localFs);
fVocabSrc = HadoopAlign.loadVocab(new Path(VOCABDIR+"/vocab.de-en.de"), localFs);
fVocabTrg = HadoopAlign.loadVocab(new Path(VOCABDIR+"/vocab.en-de.de"), localFs);