Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MappingCharFilter

77
78
79
80
81
82
83
      parseRules( wlist, normMap );
    }
  }

  public CharStream create(CharStream input) {
    return new MappingCharFilter(normMap,input);
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    MappingCharFilterFactory cfFactory = new MappingCharFilterFactory();
    List<String> mappingRules = new ArrayList<String>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap normMap = new NormalizeCharMap();
    cfFactory.parseRules( mappingRules, normMap );
    CharStream charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) );

    // create PatternTokenizer
    Map<String,String> args = new HashMap<String, String>();
    args.put( PatternTokenizerFactory.PATTERN, "[,;/\\s]+" );
    PatternTokenizerFactory tokFactory = new PatternTokenizerFactory();
    tokFactory.init( args );
    TokenStream stream = tokFactory.create( charStream );
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther", "is", "here" },
        new int[] { 0, 13, 26, 29 },
        new int[] { 12, 25, 28, 33 });
   
    charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) );
    args.put( PatternTokenizerFactory.PATTERN, "Günther" );
    args.put( PatternTokenizerFactory.GROUP, "0" );
    tokFactory = new PatternTokenizerFactory();
    tokFactory.init( args );
    stream = tokFactory.create( charStream );
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    MappingCharFilterFactory cfFactory = new MappingCharFilterFactory();
    List<String> mappingRules = new ArrayList<String>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap normMap = new NormalizeCharMap();
    cfFactory.parseRules( mappingRules, normMap );
    CharStream charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) );

    // create PatternTokenizer
    Map<String,String> args = new HashMap<String, String>();
    args.put( PatternTokenizerFactory.PATTERN, "[,;/\\s]+" );
    PatternTokenizerFactory tokFactory = new PatternTokenizerFactory();
    tokFactory.init( args );
    TokenStream stream = tokFactory.create( charStream );

    List<Token> result = getTokens( stream );
    List<Token> expect = tokens( "Günther,1,0,12 Günther,1,13,25 is,1,26,28 here,1,29,33" );
    assertTokEqualOff( expect, result );
   
    charStream.reset();
    args.put( PatternTokenizerFactory.PATTERN, "Günther" );
    args.put( PatternTokenizerFactory.GROUP, "0" );
    tokFactory = new PatternTokenizerFactory();
    tokFactory.init( args );
    stream = tokFactory.create( charStream );
View Full Code Here
51
52
53
54
55
56
57
        normMap = new NormalizeCharMap();
        parseRules(rules, normMap);
    }
   
    @Override public CharStream create(CharStream tokenStream) {
        return new MappingCharFilter(normMap, tokenStream);
    }
View Full Code Here
69
70
71
72
73
74
75
      parseRules( wlist, normMap );
    }
  }

  public CharStream create(CharStream input) {
    return new MappingCharFilter(normMap,input);
  }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
  public void testNormalizeWinDelimToLinuxDelim() throws Exception {
    NormalizeCharMap normMap = new NormalizeCharMap();
    normMap.add("\\", "/");
    String path = "c:\\a\\b\\c";
    CharStream cs = new MappingCharFilter(normMap, new StringReader(path));
    PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
    assertTokenStreamContents(t,
        new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
        new int[]{0, 0, 0, 0},
        new int[]{2, 4, 6, 8},
View Full Code Here
68
69
70
71
72
73
74
      parseRules( wlist, normMap );
    }
  }

  public CharStream create(CharStream input) {
    return new MappingCharFilter( normMap, input );
  }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
  public void testNormalizeWinDelimToLinuxDelim() throws Exception {
    NormalizeCharMap normMap = new NormalizeCharMap();
    normMap.add("\\", "/");
    String path = "c:\\a\\b\\c";
    CharStream cs = new MappingCharFilter(normMap, new StringReader(path));
    PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
    assertTokenStreamContents(t,
        new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
        new int[]{0, 0, 0, 0},
        new int[]{2, 4, 6, 8},
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
  public void testNormalizeWinDelimToLinuxDelim() throws Exception {
    NormalizeCharMap normMap = new NormalizeCharMap();
    normMap.add("\\", "/");
    String path = "c:\\a\\b\\c";
    CharStream cs = new MappingCharFilter(normMap, new StringReader(path));
    PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
    assertTokenStreamContents(t,
        new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
        new int[]{0, 0, 0, 0},
        new int[]{2, 4, 6, 8},
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
        return new TokenStreamComponents(tokenizer, filter);
      }

      //@Override
      protected Reader initReader(Reader reader) {
        return new MappingCharFilter(normMap, CharReader.get(reader));
      }
    };

    assertAnalyzesTo(analyzer, "banküberfall",
        new String[] { "bankueberfall", "fall" },
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.MappingCharFilter

  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.path.TestPathHierarchyTokenizer
  • org.apache.solr.analysis.MappingCharFilterFactory
  • org.apache.solr.analysis.TestPatternTokenizerFactory
  • org.elasticsearch.index.analysis.MappingCharFilterFactory
  • java.util.LinkedList
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.