Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termBuffer()

Package org.apache.lucene.analysis.tokenattributes

Class org.apache.lucene.analysis.tokenattributes.TermAttribute

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termBuffer()

org.apache.lucene.analysis.tokenattributes.TermAttribute.termBuffer()
Returns the internal termBuffer character array which you can then directly alter. If the array is too small for your token, use {@link #resizeTermBuffer(int)} to increase it. Afteraltering the buffer be sure to call {@link #setTermLength} to record the number of validcharacters that were placed into the termBuffer.

    TokenStream ts = analyzer.tokenStream("", reader);
    
    List<String> coll = new ArrayList<String>();
    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
    while (ts.incrementToken()) {
      char[] termBuffer = termAtt.termBuffer();
      int termLen = termAtt.termLength();
      String val = new String(termBuffer, 0, termLen);
      coll.add(val);
    }
    return coll.toArray(new String[coll.size()]);

View Full Code Here

      document = StringEscapeUtils.unescapeHtml(WikipediaDatasetCreatorMapper.CLOSE_TEXT_TAG_PATTERN.matcher(
          WikipediaDatasetCreatorMapper.OPEN_TEXT_TAG_PATTERN.matcher(document).replaceFirst("")).replaceAll(""));
      TokenStream stream = analyzer.tokenStream(catMatch, new StringReader(document));
      TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
      while (stream.incrementToken()) {
        contents.append(termAtt.termBuffer(), 0, termAtt.termLength()).append(' ');
      }
      context.write(
          new Text(WikipediaDatasetCreatorMapper.SPACE_NON_ALPHA_PATTERN.matcher(catMatch).replaceAll("_")),
          new Text(contents.toString()));
    }

View Full Code Here

                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();


                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();


                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        String rawValue = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();


                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        String rawValue = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

                } else if (f.tokenStreamValue() != null && f.tokenStreamValue() instanceof SingletonTokenStream) {
                    TokenStream tokenStream = f.tokenStreamValue();
                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                    tokenStream.incrementToken();
                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                    tokenStream.reset();
                    field = new Field(f.name(), new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone()));
                }
                if (field != null) {
                    field.setOmitNorms(f.getOmitNorms());

View Full Code Here

                } else if (f.tokenStreamValue() != null && f.tokenStreamValue() instanceof SingletonTokenStream) {
                    TokenStream tokenStream = f.tokenStreamValue();
                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                    tokenStream.incrementToken();
                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                    tokenStream.reset();
                    field = new Field(f.name(), new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone()));
                }
                if (field != null) {
                    field.setOmitNorms(f.getOmitNorms());

View Full Code Here

                } else if (f.tokenStreamValue() != null && f.tokenStreamValue() instanceof SingletonTokenStream) {
                    TokenStream tokenStream = f.tokenStreamValue();
                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                    tokenStream.incrementToken();
                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                    tokenStream.reset();
                    field = new Field(f.name(), new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone()));
                }
                if (field != null) {
                    field.setOmitNorms(f.getOmitNorms());

View Full Code Here

                                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                                    tokenStream.incrementToken();
                                    tokenStream.end();
                                    tokenStream.close();


                                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                                    if (value.startsWith(namePrefix)) {
                                        // extract value
                                        value = value.substring(namePrefix.length());
                                        // create new named value
                                        Path p = getRelativePath(state, propState);

View Full Code Here

                } else if (f.tokenStreamValue() != null && f.tokenStreamValue() instanceof SingletonTokenStream) {
                    TokenStream tokenStream = f.tokenStreamValue();
                    TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class);
                    PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class);
                    tokenStream.incrementToken();
                    String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength());
                    tokenStream.reset();
                    field = new Field(f.name(), new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone()));
                }
                if (field != null) {
                    field.setOmitNorms(f.getOmitNorms());

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.