Package org.apache.lucene.analysis.charfilter

Examples of org.apache.lucene.analysis.charfilter.MappingCharFilter

213
214
215
216
217
218
219
220
221
222
223
        return new TokenStreamComponents(tokenizer, new CJKBigramFilter(tokenizer));
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(norm, reader);
      }
    };
   
    assertAnalyzesTo(analyzer, "ab",
        new String[] { "一二", "二二", "二三" },
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        reader = new MockCharFilter(reader, 0);
        reader = new MappingCharFilter(map, reader);
        return reader;
      }
    };
    checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
  }
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
        return new TokenStreamComponents(tokenizer, filter);
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(normMap, reader);
      }
    };

    assertAnalyzesTo(analyzer, "banküberfall",
        new String[] { "bankueberfall", "fall" },
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        reader = new MockCharFilter(reader, 0);
        reader = new MappingCharFilter(map, reader);
        return reader;
      }
    };
    checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
  public void testNormalizeWinDelimToLinuxDelim() throws Exception {
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("\\", "/");
    NormalizeCharMap normMap = builder.build();
    String path = "c:\\a\\b\\c";
    Reader cs = new MappingCharFilter(normMap, new StringReader(path));
    PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
    assertTokenStreamContents(t,
        new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
        new int[]{0, 0, 0, 0},
        new int[]{2, 4, 6, 8},
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    List<String> mappingRules = new ArrayList<>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("&uuml;", "ü");
    NormalizeCharMap normMap = builder.build();
    CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );

    // create PatternTokenizer
    TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther", "is", "here" },
        new int[] { 0, 13, 26, 29 },
        new int[] { 12, 25, 28, 33 },
        INPUT.length());
   
    charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
    stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther" },
        new int[] { 0, 13 },
        new int[] { 12, 25 },
View Full Code Here
330
331
332
333
334
335
336
337
338
339
340
        return new TokenStreamComponents(tokenizer, filter);
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(normMap, reader);
      }
    };

    assertAnalyzesTo(analyzer, "banküberfall",
        new String[] { "bankueberfall", "fall" },
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
        return new TokenStreamComponents(tokenizer, new CJKBigramFilter(tokenizer));
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(norm, reader);
      }
    };
   
    assertAnalyzesTo(analyzer, "ab",
        new String[] { "一二", "二二", "二三" },
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
      builder.add("\u00AB", " | ");
      builder.add("\u00BB", " | ");

      NormalizeCharMap normMap = builder.build();

      Reader reader2 = (normMap == null ? reader : new MappingCharFilter(normMap,reader));
         
    final Tokenizer source = new WhitespaceTokenizer(matchVersion, reader2);
      TokenStream tokenStream = new LowerCaseFilter(matchVersion, source);
      tokenStream = new ShingleFilter(tokenStream, 2, 3);
      tokenStream = new NTermStopFilter(matchVersion, tokenStream, ntermStopFilterRules);   
View Full Code Here
53
54
55
56
57
58
59
        normMap = normMapBuilder.build();
    }

    @Override
    public Reader create(Reader tokenStream) {
        return new MappingCharFilter(normMap, tokenStream);
    }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.charfilter.MappingCharFilter

  • org.apache.lucene.analysis.charfilter.MappingCharFilterFactory
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.nterm.NTermAnalyzer
  • org.apache.lucene.analysis.path.TestPathHierarchyTokenizer
  • org.apache.lucene.analysis.pattern.TestPatternTokenizer
  • org.apache.lucene.util.CharsRef
  • org.elasticsearch.index.analysis.MappingCharFilterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.