// uppercase token matching features : find uppercased tokens that exactly appear on both sides

      // lack of this evidence does not imply anything, but its existence might indicate parallel

//      fSentence.replaceAll("([',:;.?%!])", " $1 ");

//      eSentence.replaceAll("([',:;.?%!])", " $1 ");

    }

    if (featSet > 3) {

      // future work = count number of single/double letter words in src and trg side

    HashSet<String> upperCaseMap1 = getUppercaseParts(tokens1);

    HashSet<String> upperCaseMap2 = getUppercaseParts(tokens2);

    float cntUpperRatio1 = getRatio(upperCaseMap1, upperCaseMap2);

    float cntUpperRatio2 = getRatio(upperCaseMap2, upperCaseMap1);

    return result;

  }

                if ( !term.equals("NULL") ) {

                  float prob = Float.parseFloat(m2.group(2));

                  int engIndex = trgVocab.addOrGet(term);

                  logger.debug("Added: "+term+" with index: "+engIndex+" and prob:"+prob);

                  sumOfProbs += prob;

                }

              }

            }

            // if number of translations not set, we never cut-off, so all cases are long tails

                if (!term.equals("NULL")) {

                  float prob = Float.parseFloat(m2.group(2));

                  int engIndex = trgVocab.addOrGet(term);

                  logger.debug("Added: "+term+" with index: "+engIndex+" and prob:"+prob);

                  sumOfProbs+=prob;

                }

              }

            }

            if(sumOfProbs > probThreshold){

    public void reduce(Text key, Iterable<PairOfIntLong> values, Context context)

        throws IOException, InterruptedException {

      String term = key.toString();

      Iterator<PairOfIntLong> iter = values.iterator();

      WritableUtils.writeVInt(dfByTermOut, df);

      WritableUtils.writeVLong(cfByTermOut, cf);

      if (iter.hasNext()) {

        throw new RuntimeException("More than one record for term: " + term);

      }

      // map from the id back to text

      // sLogger.info("termid: " + key);

      String term = mTermIdMap.getTerm(key.get());

      // sLogger.info("term: " + term);

      if (pair == null) {

        p.setCf(-1);

        p.setDf(-1);

      } else {

      }

      output.collect(key, p);

    }

      for (Entry<String> entry : probDist.entrySet()) {

        float pr = entry.getValue() / sumProb;

        if (pr > lexProbThreshold) {

          sumProb2 += pr;

        }

      }

      // re-normalize values after removal of low-prob entries

      float cumProb = 0;

      int cnt = 0;

      while (cnt < maxNumTrans && cumProb < cumProbThreshold && !sortedFilteredProbDist.isEmpty()) {

        cumProb += pr;

        cnt++;

      }

      probMap.put(sourceTerm, normProbDist);

    }

        String[] lhs = parts[0].trim().split(" ");

        String[] rhs = parts[1].trim().split(" ");;

        for (String l : lhs) {

          for (String r : rhs) {

          }

        }

      }

    } catch (UnsupportedEncodingException e) {

      e.printStackTrace();

      String eTerm = eVocab_f2e.get(e);

      //      LOG.info("Pr("+eTerm+"|"+token+")="+probEF);

        // assuming our bilingual dictionary is learned from normally segmented text, but we want to use bigram tokenizer for CLIR purposes

        // then we need to convert the translations of each source token into a sequence of bigrams

        // we can distribute the translation probability equally to the each bigram

        if (bigramSegment) {

          String[] eTokens = docLangTokenizer.processContent(eTerm);

    // in SCFG rule such as a b X1 X2 c --> X1 d e X2 f, we want to find out the src/trg tokens that are aligned to some trg/src token, ignoring the X variable

    // we can then decide if we want to include it as a multi-token phrase in our query representation based on various heuristics (e.g., only include if no X in between of tokens)

    String fPhrase = "";

    int f=0;

    for (; f < lhs.length; f++) {

      String fTerm = lhs[f];

      if (queryLangTokenizer.isStopWord(fTerm) || fTerm.matches("\\[X,\\d+\\]") || fTerm.matches("<s>") || fTerm.matches("</s>")) {

        continue;

      }

      srcTokenCnt.increment(fTerm);

      sourceTokenIds.add(f);

      if (isPassThrough){

      }else {

        ids = one2manyAlign.get(f);

      }

        continue;

      }

      // find phrase in LHS and match to phrase in RHS

      if (isMany2Many) {

        fPhrase += fTerm + " ";

        targetTokenIds = targetTokenIds.mergeNoDuplicates(ids);       

      }

      String eTerm = null;

      for (int e : ids) {

        eTerm = rhs[e];

        // assumption: if this is pass-through rule, re-stem token in doc-language

        if (isPassThrough || (unknownWords != null && unknownWords.contains(fTerm))) {

          eTerm = stemmed2Stemmed.get(eTerm);

        }

        if (eTerm == null || docLangTokenizer.isStopWord(eTerm)) {

          //          LOG.info("Skipped trg token " + eTerm);

          eTerm = null;

          continue;     

        }

        bagOfTargetTokens.add(eTerm);

        if (isOne2Many <= 1) {

          if (probDist.containsKey(fTerm)) {

            HMapSFW eToken2Prob = probDist.get(fTerm);

            eToken2Prob.increment(eTerm, weight);

          }else {

            HMapSFW eToken2Prob = new HMapSFW();

            eToken2Prob.put(eTerm, weight);

            probDist.put(fTerm, eToken2Prob);

          }

        }

      }

      if (isOne2Many == 2) {

        // if ids.size() > 1 eTerm is a multi-token expression

        // even if eTerm is overwritten here, we need to do above loop to update bagOfTargetTokens

          eTerm = isConsecutiveWithStopwords(ids, rhs, docLangTokenizer);     // <---- heuristic

        }

        // no proper translation on target-side (e.g., stopword OR non-consecutive multi-word translation), let's skip

        if (eTerm == null) {