Package org.apache.lucene.analysis.tokenattributes

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute

42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
  @Override
  public void map(Text key, Text value,
                  OutputCollector<Text,StringTuple> output, Reporter reporter) throws IOException {
    TokenStream stream = analyzer.tokenStream(key.toString(), new StringReader(value.toString()));
    TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
    StringTuple document = new StringTuple();
    while (stream.incrementToken()) {
      if (termAtt.termLength() > 0) {
        document.add(new String(termAtt.termBuffer(), 0, termAtt.termLength()));
      }
    }
    output.collect(key, document);
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
  }
 
  public void testNext() throws IOException {
   
   
    TermAttribute termAtt = (TermAttribute)merger.getAttribute(TermAttribute.class);
    OffsetAttribute offsetAtt = (OffsetAttribute)merger.getAttribute(OffsetAttribute.class);
    PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute)merger.getAttribute(PositionIncrementAttribute.class);
   
    merger.incrementToken();
   
    assertEquals("1111", termAtt.term());
    assertEquals(0, offsetAtt.startOffset());
    assertEquals(4, offsetAtt.endOffset());
    assertEquals(1, posIncAtt.getPositionIncrement());
   
    merger.incrementToken();   
    assertEquals("1111",  termAtt.term());
    assertEquals(5,  offsetAtt.startOffset());
    assertEquals(9,  offsetAtt.endOffset());
    assertEquals(1,  posIncAtt.getPositionIncrement());

    merger.incrementToken();
    assertEquals("2222",  termAtt.term());
    assertEquals(5,  offsetAtt.startOffset());
    assertEquals(9,  offsetAtt.endOffset());
    assertEquals(0,  posIncAtt.getPositionIncrement());

    merger.incrementToken();
    assertEquals("1111",  termAtt.term());
    assertEquals(10,  offsetAtt.startOffset());
    assertEquals(14,  offsetAtt.endOffset());
    assertEquals(1,  posIncAtt.getPositionIncrement());

    merger.incrementToken();
    assertEquals("3333",  termAtt.term());
    assertEquals(10,  offsetAtt.startOffset());
    assertEquals(14,  offsetAtt.endOffset());
    assertEquals(0,  posIncAtt.getPositionIncrement());

    merger.incrementToken();
    assertEquals("1111",  termAtt.term());
    assertEquals(15, offsetAtt.startOffset());
    assertEquals(19, offsetAtt.endOffset());
    assertEquals(1,  posIncAtt.getPositionIncrement());

    merger.incrementToken();
    assertEquals("2222",  termAtt.term());
    assertEquals(15,  offsetAtt.startOffset());
    assertEquals(19, offsetAtt.endOffset());
    assertEquals(0, posIncAtt.getPositionIncrement());
  }
View Full Code Here
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
                           Fieldable field = fields[k];
                           // assume properties fields use
                           // SingleTokenStream
                           //t = field.tokenStreamValue().next(t);
                           field.tokenStreamValue().incrementToken();
                           TermAttribute term = field.tokenStreamValue().getAttribute(TermAttribute.class);
                           PayloadAttribute payload = field.tokenStreamValue().getAttribute(PayloadAttribute.class);

                           String value = new String(term.termBuffer(), 0, term.termLength());

                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
                              QPath p = getRelativePath(state, propState);
                              String path = getNamespaceMappings().translatePath(p);
                              value = FieldNames.createNamedValue(path, value);

                              term.setTermBuffer(value);
                              doc.add(new Field(field.name(), new SingletonTokenStream(term.term(), payload
                                 .getPayload())));
                              doc.add(new Field(FieldNames.AGGREGATED_NODE_UUID, parent.getIdentifier(),
                                 Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                           }
                        }
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    } else {
      TokenStream tokens = analyzer.tokenStream(
                              field, new StringReader(terms.toString()));

      TermAttribute ta = tokens.getAttribute(TermAttribute.class);
      try
      {
        String termText;
        while (tokens.incrementToken())
        {
          if ((termText = ta.term()) == null)
            break;
          result.add(termText);
        }
      } catch (IOException e) {
        // ignore (?)
View Full Code Here
734
735
736
737
738
739
740
741
742
743
744
745
            throws IOException {
        TokenStream ts = analyzer.tokenStream(fieldName, r);
        int tokenCount = 0;
        // for every token
        while (ts.incrementToken()) {
            TermAttribute term = ts.getAttribute(TermAttribute.class);
            String word =  term.term();
            tokenCount++;
            if (tokenCount > maxNumTokensParsed) {
                break;
            }
            if (isNoiseWord(word)) {
View Full Code Here
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
                                Fieldable[] fields = aDoc.getFieldables(FieldNames.PROPERTIES);
                                for (Fieldable field : fields) {

                                    // assume properties fields use SingleTokenStream
                                    TokenStream tokenStream = field.tokenStreamValue();
                                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();

                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);
                                        String path = getNamespaceMappings().translatePath(p);
                                        value = FieldNames.createNamedValue(path, value);
                                        termAttribute.setTermBuffer(value);
                                        doc.add(new Field(field.name(),
                                                new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone())));
                                        doc.add(new Field(
                                                FieldNames.AGGREGATED_NODE_UUID,
                                                false,
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
          "Alternatively, apply the ignoreFieldBridge() option to " +
          "pass String parameters" );
    }
    Reader reader = new StringReader(localText);
    TokenStream stream = analyzer.reusableTokenStream( fieldName, reader);
    TermAttribute attribute = stream.addAttribute( TermAttribute.class );
    stream.reset();

    while ( stream.incrementToken() ) {
      if ( attribute.termLength() > 0 ) {
        String term = attribute.term();
        terms.add( term );
      }
    }
    stream.end();
    stream.close();
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
          "Alternatively, apply the ignoreFieldBridge() option to " +
          "pass String parameters" );
    }
    Reader reader = new StringReader(localText);
    TokenStream stream = analyzer.reusableTokenStream( fieldName, reader);
    TermAttribute attribute = stream.addAttribute( TermAttribute.class );
    stream.reset();

    while ( stream.incrementToken() ) {
      if ( attribute.termLength() > 0 ) {
        String term = attribute.term();
        terms.add( term );
      }
    }
    stream.end();
    stream.close();
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    final String sentence = phraseContext.getSentence();
    try {
      Reader reader = new StringReader( sentence );
      stream = queryContext.getQueryAnalyzer().reusableTokenStream( fieldName, reader);

      TermAttribute termAttribute = stream.addAttribute( TermAttribute.class );
      PositionIncrementAttribute positionAttribute = stream.addAttribute( PositionIncrementAttribute.class );

      stream.reset();
      int position = -1; //start at -1 since we apply at least one increment
      List<Term> termsAtSamePosition = null;
      while ( stream.incrementToken() ) {
        int positionIncrement = 1;
        if ( positionAttribute != null ) {
          positionIncrement = positionAttribute.getPositionIncrement();
        }

        if ( positionIncrement > 0 ) {
          position+=positionIncrement;
          termsAtSamePosition = termsPerPosition.get(position);
        }

        if (termsAtSamePosition == null) {
          termsAtSamePosition = new ArrayList<Term>();
          termsPerPosition.put( position, termsAtSamePosition  );
        }

        termsAtSamePosition.add( new Term( fieldName, termAttribute.term() ) );
        if ( termsAtSamePosition.size() > 1 ) {
          isMultiPhrase = true;
        }
      }
    }
View Full Code Here
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
    public boolean incrementToken() {
      if (upto == 4) {
        return false;
      }
      PositionIncrementAttribute posIncr = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
      TermAttribute term = (TermAttribute) addAttribute(TermAttribute.class);
      if (upto == 0) {
        posIncr.setPositionIncrement(1);
        term.setTermBuffer("a");
      } else if (upto == 1) {
        posIncr.setPositionIncrement(1);
        term.setTermBuffer("b");
      } else if (upto == 2) {
        posIncr.setPositionIncrement(0);
        term.setTermBuffer("c");
      } else {
        posIncr.setPositionIncrement(0);
        term.setTermBuffer("d");
      }
      upto++;
      return true;
    }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.tokenattributes.TermAttribute

  • com.alimama.mdrill.index.IndexReducer
  • com.flaptor.indextank.query.IndexEngineParser
  • com.jayway.mongodb.AnalyzedDBObject
  • com.senseidb.search.query.TextQueryConstructor
  • edu.wiki.search.ESASearcher
  • it.unibz.instasearch.indexing.StorageIndexer
  • lucli.LuceneMethods
  • mia.classifier.ch14.TokenizingAndVectorizingText
  • mia.clustering.ch09.MyAnalyzer
  • mia.clustering.ch12.TwitterAnalyzer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.