Package org.apache.lucene.analysis.tokenattributes

Examples of org.apache.lucene.analysis.tokenattributes.PayloadAttribute

1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
                                for (Fieldable field : fields) {

                                    // assume properties fields use SingleTokenStream
                                    TokenStream tokenStream = field.tokenStreamValue();
                                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();

                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);
                                        String path = getNamespaceMappings().translatePath(p);
                                        value = FieldNames.createNamedValue(path, value);
                                        termAttribute.setTermBuffer(value);
                                        doc.add(new Field(field.name(),
                                                new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone())));
                                        doc.add(new Field(
                                                FieldNames.AGGREGATED_NODE_UUID,
                                                false,
                                                parent.getNodeId().toString(),
                                                Field.Store.NO,
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
  public void testFilterTokens() throws Exception {
    SnowballFilter filter = new SnowballFilter(new TestTokenStream(), "English");
    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
    OffsetAttribute offsetAtt = filter.getAttribute(OffsetAttribute.class);
    TypeAttribute typeAtt = filter.getAttribute(TypeAttribute.class);
    PayloadAttribute payloadAtt = filter.getAttribute(PayloadAttribute.class);
    PositionIncrementAttribute posIncAtt = filter.getAttribute(PositionIncrementAttribute.class);
    FlagsAttribute flagsAtt = filter.getAttribute(FlagsAttribute.class);
   
    filter.incrementToken();

    assertEquals("accent", termAtt.toString());
    assertEquals(2, offsetAtt.startOffset());
    assertEquals(7, offsetAtt.endOffset());
    assertEquals("wrd", typeAtt.type());
    assertEquals(3, posIncAtt.getPositionIncrement());
    assertEquals(77, flagsAtt.getFlags());
    assertEquals(new Payload(new byte[]{0,1,2,3}), payloadAtt.getPayload());
  }
View Full Code Here
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
                           // assume properties fields use
                           // SingleTokenStream
                           //t = field.tokenStreamValue().next(t);
                           field.tokenStreamValue().incrementToken();
                           TermAttribute term = field.tokenStreamValue().getAttribute(TermAttribute.class);
                           PayloadAttribute payload = field.tokenStreamValue().getAttribute(PayloadAttribute.class);

                           String value = new String(term.termBuffer(), 0, term.termLength());

                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
                              QPath p = getRelativePath(state, propState);
                              String path = getNamespaceMappings().translatePath(p);
                              value = FieldNames.createNamedValue(path, value);

                              term.setTermBuffer(value);
                              doc.add(new Field(field.name(), new SingletonTokenStream(term.term(), payload
                                 .getPayload())));
                              doc.add(new Field(FieldNames.AGGREGATED_NODE_UUID, parent.getIdentifier(),
                                 Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                           }
                        }
View Full Code Here
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
                           // assume properties fields use
                           // SingleTokenStream
                           //t = field.tokenStreamValue().next(t);
                           field.tokenStreamValue().incrementToken();
                           CharTermAttribute term = field.tokenStreamValue().getAttribute(CharTermAttribute.class);
                           PayloadAttribute payload = field.tokenStreamValue().getAttribute(PayloadAttribute.class);

                           String value = new String(term.buffer(), 0, term.length());

                           if (value.startsWith(namePrefix))
                           {
                              // extract value
                              value = value.substring(namePrefix.length());
                              // create new named value
                              QPath p = getRelativePath(state, propState);
                              String path = getNamespaceMappings().translatePath(p);
                              value = FieldNames.createNamedValue(path, value);

                              term.setEmpty();
                              term.append(value);
                              doc.add(new Field(field.name(), new SingletonTokenStream(term.toString(), payload
                                 .getPayload())));
                              doc.add(new Field(FieldNames.AGGREGATED_NODE_UUID, parent.getIdentifier(),
                                 Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                           }
                        }
View Full Code Here
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
                                for (Fieldable field : fields) {

                                    // assume properties fields use SingleTokenStream
                                    TokenStream tokenStream = field.tokenStreamValue();
                                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();

                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);
                                        String path = getNamespaceMappings().translatePath(p);
                                        value = FieldNames.createNamedValue(path, value);
                                        termAttribute.setTermBuffer(value);
                                        doc.add(new Field(field.name(),
                                                new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone())));
                                        doc.add(new Field(
                                                FieldNames.AGGREGATED_NODE_UUID,
                                                false,
                                                parent.getNodeId().toString(),
                                                Field.Store.NO,
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
315
      record.put( "sequence", charAttr.toString() );
      return record;
    }
    else if (attr instanceof PayloadAttribute) {
      GenericRecord record = new GenericData.Record( protocol.getType( "PayloadAttribute" ) );
      PayloadAttribute payloadAttr = (PayloadAttribute) attr;
      record.put( "payload", ByteBuffer.wrap( payloadAttr.getPayload().toByteArray() ) );
      return record;
    }
    else if (attr instanceof KeywordAttribute) {
      GenericRecord record = new GenericData.Record( protocol.getType( "KeywordAttribute" ) );
      KeywordAttribute narrowedAttr = (KeywordAttribute) attr;
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
307
      record.put( "sequence", charAttr.toString() );
      return record;
    }
    else if ( attr instanceof PayloadAttribute ) {
      GenericRecord record = new GenericData.Record( protocol.getType( "PayloadAttribute" ) );
      PayloadAttribute payloadAttr = (PayloadAttribute) attr;
      BytesRef payload = payloadAttr.getPayload();
      record.put( "payload", ByteBuffer.wrap( payload.bytes, payload.offset, payload.length ) );
      return record;
    }
    else if ( attr instanceof KeywordAttribute ) {
      GenericRecord record = new GenericData.Record( protocol.getType( "KeywordAttribute" ) );
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
        allEntries.addText("field1", "boosts", 0.5f);
        allEntries.reset();
        // whitespace analyzer's tokenizer reads characters eagerly on the contrary to the standard tokenizer
        final TokenStream ts = AllTokenStream.allTokenStream("any", allEntries, new WhitespaceAnalyzer());
        final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
        final PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
        ts.reset();
        for (int i = 0; i < 3; ++i) {
            assertTrue(ts.incrementToken());
            final String term;
            final float boost;
            switch (i) {
            case 0:
                term = "all";
                boost = 2;
                break;
            case 1:
                term = "your";
                boost = 1;
                break;
            case 2:
                term = "boosts";
                boost = 0.5f;
                break;
            default:
                throw new AssertionError();
            }
            assertEquals(term, termAtt.toString());
            final BytesRef payload = payloadAtt.getPayload();
            if (payload == null || payload.length == 0) {
                assertEquals(boost, 1f, 0.001f);
            } else {
                assertEquals(4, payload.length);
                final float b = PayloadHelper.decodeFloat(payload.bytes, payload.offset);
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    assertAnalyzesTo(a, "foo a bar such FOO THESE BAR",
                     new String[] { "foo", "bar", "foo", "bar" });
  }

  void verifyPayload(TokenStream ts) throws IOException {
    PayloadAttribute payloadAtt = ts.getAttribute(PayloadAttribute.class);
    for(byte b=1;;b++) {
      boolean hasNext = ts.incrementToken();
      if (!hasNext) break;
      // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
      // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
      assertEquals(b, payloadAtt.getPayload().toByteArray()[0]);
    }
  }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
    String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
    DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
      (new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)),
       DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
    PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
    assertTermEquals("The", filter, termAtt, payAtt, null);
    assertTermEquals("quick", filter, termAtt, payAtt, "JJ".getBytes("UTF-8"));
    assertTermEquals("red", filter, termAtt, payAtt, "JJ".getBytes("UTF-8"));
    assertTermEquals("fox", filter, termAtt, payAtt, "NN".getBytes("UTF-8"));
    assertTermEquals("jumped", filter, termAtt, payAtt, "VB".getBytes("UTF-8"));
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.tokenattributes.PayloadAttribute

  • com.github.le11.nls.lucene.UIMAPayloadsAnalyzerTest
  • org.apache.jackrabbit.core.query.lucene.AbstractIndex
  • org.apache.jackrabbit.core.query.lucene.SearchIndex
  • org.apache.lucene.analysis.core.TestAnalyzers
  • org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterTest
  • org.apache.lucene.analysis.payloads.NumericPayloadTokenFilterTest
  • org.apache.lucene.analysis.payloads.TestDelimitedPayloadTokenFilterFactory
  • org.apache.lucene.analysis.payloads.TokenOffsetPayloadTokenFilterTest
  • org.apache.lucene.analysis.payloads.TypeAsPayloadTokenFilterTest
  • org.apache.lucene.analysis.shingle.TestShingleMatrixFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.