Package org.apache.lucene.analysis.charfilter

Examples of org.apache.lucene.analysis.charfilter.MappingCharFilter

61
62
63
64
65
66
67
68
69
70
71
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        reader = new MockCharFilter(reader, 0);
        reader = new MappingCharFilter(map, reader);
        return reader;
      }
    };
    checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    List<String> mappingRules = new ArrayList<String>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("&uuml;", "ü");
    NormalizeCharMap normMap = builder.build();
    CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );

    // create PatternTokenizer
    TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther", "is", "here" },
        new int[] { 0, 13, 26, 29 },
        new int[] { 12, 25, 28, 33 },
        INPUT.length());
   
    charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
    stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther" },
        new int[] { 0, 13 },
        new int[] { 12, 25 },
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        reader = new MockCharFilter(reader, 0);
        reader = new MappingCharFilter(map, reader);
        return reader;
      }
    };
    checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    List<String> mappingRules = new ArrayList<String>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("&uuml;", "ü");
    NormalizeCharMap normMap = builder.build();
    CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );

    // create PatternTokenizer
    TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther", "is", "here" },
        new int[] { 0, 13, 26, 29 },
        new int[] { 12, 25, 28, 33 },
        INPUT.length());
   
    charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
    stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther" },
        new int[] { 0, 13 },
        new int[] { 12, 25 },
View Full Code Here
80
81
82
83
84
85
86
  @Override
  public Reader create(Reader input) {
    // if the map is null, it means there's actually no mappings... just return the original stream
    // as there is nothing to do here.
    return normMap == null ? input : new MappingCharFilter(normMap,input);
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
    List<String> mappingRules = new ArrayList<>();
    mappingRules.add( "\"&uuml;\" => \"ü\"" );
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("&uuml;", "ü");
    NormalizeCharMap normMap = builder.build();
    CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );

    // create PatternTokenizer
    Tokenizer stream = new PatternTokenizer(newAttributeFactory(), charStream, Pattern.compile("[,;/\\s]+"), -1);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther", "is", "here" },
        new int[] { 0, 13, 26, 29 },
        new int[] { 12, 25, 28, 33 },
        INPUT.length());
   
    charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
    stream = new PatternTokenizer(newAttributeFactory(), charStream, Pattern.compile("Günther"), 0);
    assertTokenStreamContents(stream,
        new String[] { "Günther", "Günther" },
        new int[] { 0, 13 },
        new int[] { 12, 25 },
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
        return new TokenStreamComponents(tokenizer, filter);
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(normMap, reader);
      }
    };

    assertAnalyzesTo(analyzer, "banküberfall",
        new String[] { "bankueberfall", "fall" },
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
  public void testNormalizeWinDelimToLinuxDelim() throws Exception {
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add("\\", "/");
    NormalizeCharMap normMap = builder.build();
    String path = "c:\\a\\b\\c";
    Reader cs = new MappingCharFilter(normMap, new StringReader(path));
    PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), cs, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
    assertTokenStreamContents(t,
        new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
        new int[]{0, 0, 0, 0},
        new int[]{2, 4, 6, 8},
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        reader = new MockCharFilter(reader, 0);
        reader = new MappingCharFilter(map, reader);
        return reader;
      }
    };
    checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
  }
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
        return new TokenStreamComponents(tokenizer, filter);
      }

      @Override
      protected Reader initReader(String fieldName, Reader reader) {
        return new MappingCharFilter(normMap, reader);
      }
    };

    assertAnalyzesTo(analyzer, "banküberfall",
        new String[] { "bankueberfall", "fall" },
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.charfilter.MappingCharFilter

  • org.apache.lucene.analysis.charfilter.MappingCharFilterFactory
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.nterm.NTermAnalyzer
  • org.apache.lucene.analysis.path.TestPathHierarchyTokenizer
  • org.apache.lucene.analysis.pattern.TestPatternTokenizer
  • org.apache.lucene.util.CharsRef
  • org.elasticsearch.index.analysis.MappingCharFilterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.