Package ivory.core.tokenize

Examples of ivory.core.tokenize.BigramChineseTokenizer

62
63
64
65
66
67
68
69
70
71
72
    cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    float cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    }else if (lang.equals(Constants.German)) {
      tokenizer = TokenizerFactory.createTokenizer(conf.get(Constants.DocLanguage), conf.get(Constants.DocTokenizerData), null);
    }else if (lang.equals(Constants.Chinese)) {
      if (!fs.exists(new Path(tokenizerPath))) {
        LOG.info("Tokenizer path "+tokenizerPath+" doesn't exist -- using BigramChineseTokenizer");
        tokenizer = new BigramChineseTokenizer();
      }else {
        tokenizer = TokenizerFactory.createTokenizer(conf.get(Constants.DocLanguage), conf.get(Constants.DocTokenizerData), null);
      }
    }else {
      throw new RuntimeException("DocLanguage code "+lang+ " not known");
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    float cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    }else if (lang.equals(Constants.German)) {
      tokenizer = TokenizerFactory.createTokenizer(lang, tokenizerPath, true);
    }else if (lang.equals(Constants.Chinese)) {
      if (!fs.exists(new Path(tokenizerPath))) {
        LOG.info("Tokenizer path "+tokenizerPath+" doesn't exist -- using BigramChineseTokenizer");
        tokenizer = new BigramChineseTokenizer();
      }else {
        tokenizer = TokenizerFactory.createTokenizer(lang, tokenizerPath, true);
      }
    }else {
      throw new RuntimeException("DocLanguage code "+lang+ " not known");
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    float cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
    }else if (lang.equals(Constants.German)) {
      tokenizer = TokenizerFactory.createTokenizer(conf.get(Constants.DocLanguage), conf.get(Constants.DocTokenizerData), null);
    }else if (lang.equals(Constants.Chinese)) {
      if (!fs.exists(new Path(tokenizerPath))) {
        LOG.info("Tokenizer path "+tokenizerPath+" doesn't exist -- using BigramChineseTokenizer");
        tokenizer = new BigramChineseTokenizer();
      }else {
        tokenizer = TokenizerFactory.createTokenizer(conf.get(Constants.DocLanguage), conf.get(Constants.DocTokenizerData), null);
      }
    }else {
      throw new RuntimeException("DocLanguage code "+lang+ " not known");
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    }else if (lang.equals(Constants.German)) {
      tokenizer = TokenizerFactory.createTokenizer(lang, tokenizerPath, true);
    }else if (lang.equals(Constants.Chinese)) {
      if (!fs.exists(new Path(tokenizerPath))) {
        LOG.info("Tokenizer path "+tokenizerPath+" doesn't exist -- using BigramChineseTokenizer");
        tokenizer = new BigramChineseTokenizer();
      }else {
        tokenizer = TokenizerFactory.createTokenizer(lang, tokenizerPath, true);
      }
    }else {
      throw new RuntimeException("DocLanguage code "+lang+ " not known");
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    cumProbThreshold = conf.getFloat(Constants.CumulativeProbThreshold, 1f);

    String bigram = conf.get(Constants.BigramSegment);
    bigramSegment = (bigram != null && bigram.equals("on")) ? true : false;
    if (bigramSegment) {
      bigramTokenizer = new BigramChineseTokenizer();
    }
    LOG.info("Bigram segmentation = " + bigramSegment);

    // initialize environment to access index
    try {
View Full Code Here
TOP

Related Classes of ivory.core.tokenize.BigramChineseTokenizer

  • ivory.sqe.querygenerator.BagOfWordsQueryGenerator
  • ivory.sqe.querygenerator.DefaultBagOfWordQueryGenerator
  • ivory.sqe.querygenerator.SCFGQueryGenerator
  • java.io.InputStreamReader
  • java.io.FileInputStream
  • java.io.BufferedReader
  • java.io.OutputStreamWriter
  • java.io.FileOutputStream
  • java.io.BufferedWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.