Examples of etc.aloe.filters.SimpleStringToWordVector$NoNonsenseStemmer

etc.aloe.filters.SimpleStringToWordVector
Extension of Weka's StringToWordVector filter that allows a string attribute to be specified by name, and uses a special term filter (NoNonsenseStemmer). @author Michael Brooks

     * @return
     * @throws Exception
     */
    @Override
    protected Filter getBagOfWordsFilter(ExampleSet examples) throws Exception {
        SimpleStringToWordVector filter = new SimpleStringToWordVector();
        filter.setAttributeNamePrefix(BAG_OF_WORDS_FEATURE_PREFIX);
        filter.setStringAttributeName(ExampleSet.MESSAGE_ATTR_NAME);


        //This is stupid because it depends on how much data you use
        //bagger.setMinTermFreq(20);
        
        filter.setDoNotOperateOnPerClassBasis(true);
        filter.setWordsToKeep(3000);
        filter.setLowerCaseTokens(true);


        //use stemming and remove "nonsense"
        filter.setStemmer(new NoNonsenseStemmer(true));


        filter.setTFTransform(true);
        filter.setIDFTransform(true);
        filter.setNormalizeDocLength(new SelectedTag(StringToWordVector.FILTER_NORMALIZE_ALL, StringToWordVector.TAGS_FILTER));


        filter.setOutputWordCounts(true);


        filter.setInputFormat(examples.getInstances());
        Instances filtered = Filter.useFilter(examples.getInstances(), filter);
        examples.setInstances(filtered);


        return filter;
    }

View Full Code Here

     * @param examples
     * @return
     * @throws Exception
     */
    protected Filter getBagOfWordsFilter(ExampleSet examples) throws Exception {
        SimpleStringToWordVector filter = new SimpleStringToWordVector();
        filter.setAttributeNamePrefix(BAG_OF_WORDS_FEATURE_PREFIX);
        filter.setStringAttributeName(ExampleSet.MESSAGE_ATTR_NAME);


        //This is stupid because it depends on how much data you use
        //bagger.setMinTermFreq(20);
        
        filter.setDoNotOperateOnPerClassBasis(true);
        filter.setWordsToKeep(800);
        filter.setLowerCaseTokens(true);


        //use stemming and remove "nonsense"
        filter.setStemmer(new NoNonsenseStemmer(true));


        filter.setTFTransform(true);
        filter.setIDFTransform(true);
        filter.setNormalizeDocLength(new SelectedTag(StringToWordVector.FILTER_NORMALIZE_ALL, StringToWordVector.TAGS_FILTER));


        filter.setOutputWordCounts(true);


        filter.setInputFormat(examples.getInstances());
        Instances filtered = Filter.useFilter(examples.getInstances(), filter);
        examples.setInstances(filtered);


        return filter;
    }

View Full Code Here

     * @param examples
     * @return
     * @throws Exception
     */
    protected Filter getParticipantsFilter(ExampleSet examples) throws Exception {
        SimpleStringToWordVector filter = new SimpleStringToWordVector();
        filter.setAttributeNamePrefix(PARTICIPANT_FEATURE_PREFIX);
        filter.setStringAttributeName(ExampleSet.PARTICIPANT_ATTR_NAME);


        filter.setDoNotOperateOnPerClassBasis(true);
        filter.setWordsToKeep(getParticipantFeatureCount());
        filter.setLowerCaseTokens(true);


        //use stemming and remove "nonsense"
        filter.setStemmer(null);
        
        filter.setOutputWordCounts(false);


        filter.setInputFormat(examples.getInstances());
        Instances filtered = Filter.useFilter(examples.getInstances(), filter);
        examples.setInstances(filtered);


        return filter;
    }

View Full Code Here

TOP

Related Classes of etc.aloe.filters.SimpleStringToWordVector$NoNonsenseStemmer

etc.aloe.cscw2013.FeatureGenerationImpl

etc.aloe.oilspill2010.FeatureGenerationImpl

weka.core.Attribute

weka.core.stemmers.SnowballStemmer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.