/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package opennlp.tools.doccat;
import java.io.IOException;
import java.io.ObjectStreamException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import opennlp.tools.ml.model.MaxentModel;
import opennlp.tools.ml.model.TrainUtil;
import opennlp.tools.tokenize.SimpleTokenizer;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
/**
* Maxent implementation of {@link DocumentCategorizer}.
*/
public class DocumentCategorizerME implements DocumentCategorizer {
/**
* Shared default thread safe feature generator.
*/
private static FeatureGenerator defaultFeatureGenerator = new BagOfWordsFeatureGenerator();
private DoccatModel model;
private DocumentCategorizerContextGenerator mContextGenerator;
/**
* Initializes a the current instance with a doccat model and custom feature
* generation. The feature generation must be identical to the configuration
* at training time.
*
* @param model
* @param featureGenerators
*
* @deprecated train a {@link DoccatModel} with a specific
* {@link DoccatFactory} to customize the {@link FeatureGenerator}s
*/
public DocumentCategorizerME(DoccatModel model, FeatureGenerator... featureGenerators) {
this.model = model;
this.mContextGenerator = new DocumentCategorizerContextGenerator(featureGenerators);
}
/**
* Initializes the current instance with a doccat model. Default feature
* generation is used.
*
* @param model
*/
public DocumentCategorizerME(DoccatModel model) {
this.model = model;
this.mContextGenerator = new DocumentCategorizerContextGenerator(this.model
.getFactory().getFeatureGenerators());
}
@Override
public double[] categorize(String[] text, Map<String, Object> extraInformation) {
return model.getMaxentModel().eval(
mContextGenerator.getContext(text, extraInformation));
}
/**
* Categorizes the given text.
*
* @param text
*/
public double[] categorize(String text[]) {
return this.categorize(text, Collections.<String, Object>emptyMap());
}
/**
* Categorizes the given text. The Tokenizer is obtained from
* {@link DoccatFactory#getTokenizer()} and defaults to
* {@link SimpleTokenizer}.
*/
@Override
public double[] categorize(String documentText,
Map<String, Object> extraInformation) {
Tokenizer tokenizer = model.getFactory().getTokenizer();
return categorize(tokenizer.tokenize(documentText), extraInformation);
}
/**
* Categorizes the given text. The text is tokenized with the SimpleTokenizer
* before it is passed to the feature generation.
*/
public double[] categorize(String documentText) {
Tokenizer tokenizer = model.getFactory().getTokenizer();
return categorize(tokenizer.tokenize(documentText),
Collections.<String, Object> emptyMap());
}
/**
* Returns a map in which the key is the category name and the value is the score
* @param text the input text to classify
* @return
*/
public Map<String, Double> scoreMap(String text) {
Map<String, Double> probDist = new HashMap<String, Double>();
double[] categorize = categorize(text);
int catSize = getNumberOfCategories();
for (int i = 0; i < catSize; i++) {
String category = getCategory(i);
probDist.put(category, categorize[getIndex(category)]);
}
return probDist;
}
/**
* Returns a map with the score as a key in ascendng order. The value is a Set of categories with the score.
* Many categories can have the same score, hence the Set as value
* @param text the input text to classify
* @return
*/
public SortedMap<Double, Set<String>> sortedScoreMap(String text) {
SortedMap<Double, Set<String>> descendingMap = new TreeMap<Double, Set<String>>();
double[] categorize = categorize(text);
int catSize = getNumberOfCategories();
for (int i = 0; i < catSize; i++) {
String category = getCategory(i);
double score = categorize[getIndex(category)];
if (descendingMap.containsKey(score)) {
descendingMap.get(score).add(category);
} else {
Set<String> newset = new HashSet<>();
newset.add(category);
descendingMap.put(score, newset);
}
}
return descendingMap;
}
public String getBestCategory(double[] outcome) {
return model.getMaxentModel().getBestOutcome(outcome);
}
public int getIndex(String category) {
return model.getMaxentModel().getIndex(category);
}
public String getCategory(int index) {
return model.getMaxentModel().getOutcome(index);
}
public int getNumberOfCategories() {
return model.getMaxentModel().getNumOutcomes();
}
public String getAllResults(double results[]) {
return model.getMaxentModel().getAllOutcomes(results);
}
/**
* @deprecated Use
* {@link #train(String, ObjectStream, TrainingParameters, DoccatFactory)}
* instead.
*/
public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples,
TrainingParameters mlParams, FeatureGenerator... featureGenerators)
throws IOException {
if (featureGenerators.length == 0) {
featureGenerators = new FeatureGenerator[]{defaultFeatureGenerator};
}
Map<String, String> manifestInfoEntries = new HashMap<String, String>();
MaxentModel model = TrainUtil.train(
new DocumentCategorizerEventStream(samples, featureGenerators),
mlParams.getSettings(), manifestInfoEntries);
return new DoccatModel(languageCode, model, manifestInfoEntries);
}
public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples,
TrainingParameters mlParams, DoccatFactory factory)
throws IOException {
Map<String, String> manifestInfoEntries = new HashMap<String, String>();
MaxentModel model = TrainUtil.train(
new DocumentCategorizerEventStream(samples, factory.getFeatureGenerators()),
mlParams.getSettings(), manifestInfoEntries);
return new DoccatModel(languageCode, model, manifestInfoEntries, factory);
}
/**
* Trains a doccat model with default feature generation.
*
* @param languageCode
* @param samples
*
* @return the trained doccat model
*
* @throws IOException
* @throws ObjectStreamException
*
* @deprecated Use
* {@link #train(String, ObjectStream, TrainingParameters, DoccatFactory)}
* instead.
*/
public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples) throws IOException {
return train(languageCode, samples, ModelUtil.createDefaultTrainingParameters(), defaultFeatureGenerator);
}
}