/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
import static org.apache.jackrabbit.oak.commons.PathUtils.getAncestorPath;
import static org.apache.jackrabbit.oak.commons.PathUtils.getDepth;
import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newFulltextTerm;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm;
import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization;
import static org.apache.jackrabbit.oak.query.QueryImpl.JCR_PATH;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import javax.jcr.PropertyType;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper;
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
import org.apache.jackrabbit.oak.query.QueryImpl;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.query.fulltext.FullTextOr;
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.spi.query.Cursor;
import org.apache.jackrabbit.oak.spi.query.Cursors.PathCursor;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction;
import org.apache.jackrabbit.oak.spi.query.IndexRow;
import org.apache.jackrabbit.oak.spi.query.PropertyValues;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides a QueryIndex that does lookups against a Lucene-based index
*
* <p>
* To define a lucene index on a subtree you have to add an
* <code>oak:index<code> node.
*
* Under it follows the index definition node that:
* <ul>
* <li>must be of type <code>oak:QueryIndexDefinition</code></li>
* <li>must have the <code>type</code> property set to <b><code>lucene</code></b></li>
* <li>must have the <code>async</code> property set to <b><code>async</code></b></li>
* </b></li>
* </ul>
* </p>
* <p>
* Optionally you can add
* <ul>
* <li>what subset of property types to be included in the index via the <code>includePropertyTypes<code> property</li>
* <li>a blacklist of property names: what property to be excluded from the index via the <code>excludePropertyNames<code> property</li>
* <li>the <code>reindex<code> flag which when set to <code>true<code>, triggers a full content re-index.</li>
* </ul>
* </p>
* <pre>
* <code>
* {
* NodeBuilder index = root.child("oak:index");
* index.child("lucene")
* .setProperty("jcr:primaryType", "oak:QueryIndexDefinition", Type.NAME)
* .setProperty("type", "lucene")
* .setProperty("async", "async")
* .setProperty("reindex", "true");
* }
* </code>
* </pre>
*
* @see QueryIndex
*
*/
public class LuceneIndex implements FulltextQueryIndex {
private static final Logger LOG = LoggerFactory
.getLogger(LuceneIndex.class);
public static final String NATIVE_QUERY_FUNCTION = "native*lucene";
/**
* Batch size for fetching results from Lucene queries.
*/
static final int LUCENE_QUERY_BATCH_SIZE = 50;
private final IndexTracker tracker;
private final Analyzer analyzer;
private final NodeAggregator aggregator;
public LuceneIndex(
IndexTracker tracker, Analyzer analyzer,
NodeAggregator aggregator) {
this.tracker = tracker;
this.analyzer = analyzer;
this.aggregator = aggregator;
}
@Override
public String getIndexName() {
return "lucene";
}
@Override
public double getCost(Filter filter, NodeState root) {
if (tracker.getIndexNode("/") == null) {
// unusable index
return Double.POSITIVE_INFINITY;
}
FullTextExpression ft = filter.getFullTextConstraint();
if (ft == null) {
// no full-text condition: don't use this index,
// as there might be a better one
return Double.POSITIVE_INFINITY;
}
Set<String> relPaths = getRelativePaths(ft);
if (relPaths.size() > 1) {
LOG.warn("More than one relative parent for query " + filter.getQueryStatement());
// there are multiple "parents", as in
// "contains(a/x, 'hello') and contains(b/x, 'world')"
return new MultiLuceneIndex(filter, root, relPaths).getCost();
}
String parent = relPaths.iterator().next();
if (parent.isEmpty()) {
// no relative properties
return 10;
}
// all relative properties have the same "parent", as in
// "contains(a/x, 'hello') and contains(a/y, 'world')" or
// "contains(a/x, 'hello') or contains(a/*, 'world')"
// TODO: proper cost calculation
// we assume this will cause more read operations,
// as we need to read the node and then the parent
return 15;
}
/**
* Get the set of relative paths of a full-text condition. For example, for
* the condition "contains(a/b, 'hello') and contains(c/d, 'world'), the set
* { "a", "c" } is returned. If there are no relative properties, then one
* entry is returned (the empty string). If there is no expression, then an
* empty set is returned.
*
* @param ft the full-text expression
* @return the set of relative paths (possibly empty)
*/
private static Set<String> getRelativePaths(FullTextExpression ft) {
if (ft == null) {
// there might be no full-text constraint when using the
// LowCostLuceneIndexProvider which is used for testing
// TODO if the LowCostLuceneIndexProvider is removed, we should do
// the following instead:
// throw new
// IllegalStateException("Lucene index is used even when no full-text conditions are used for filter "
// + filter);
return Collections.emptySet();
}
final HashSet<String> relPaths = new HashSet<String>();
ft.accept(new FullTextVisitor.FullTextVisitorBase() {
@Override
public boolean visit(FullTextTerm term) {
String p = term.getPropertyName();
if (p == null) {
relPaths.add("");
} else if (p.startsWith("../") || p.startsWith("./")) {
throw new IllegalArgumentException("Relative parent is not supported:" + p);
} else if (getDepth(p) > 1) {
String parent = getParentPath(p);
relPaths.add(parent);
} else {
relPaths.add("");
}
return true;
}
});
return relPaths;
}
@Override
public String getPlan(Filter filter, NodeState root) {
IndexNode index = tracker.getIndexNode("/");
checkState(index != null, "The Lucene index is not available");
FullTextExpression ft = filter.getFullTextConstraint();
Set<String> relPaths = getRelativePaths(ft);
if (relPaths.size() > 1) {
return new MultiLuceneIndex(filter, root, relPaths).getPlan();
}
String parent = relPaths.size() == 0 ? "" : relPaths.iterator().next();
// we only restrict non-full-text conditions if there is
// no relative property in the full-text constraint
boolean nonFullTextConstraints = parent.isEmpty();
String plan = getQuery(filter, null, nonFullTextConstraints, analyzer, index.getDefinition()) + " ft:(" + ft + ")";
if (!parent.isEmpty()) {
plan += " parent:" + parent;
}
return plan;
}
@Override
public Cursor query(final Filter filter, final NodeState root) {
final IndexNode index = tracker.getIndexNode("/");
checkState(index != null, "The Lucene index is not available");
FullTextExpression ft = filter.getFullTextConstraint();
Set<String> relPaths = getRelativePaths(ft);
if (relPaths.size() > 1) {
return new MultiLuceneIndex(filter, root, relPaths).query();
}
final String parent = relPaths.size() == 0 ? "" : relPaths.iterator().next();
// we only restrict non-full-text conditions if there is
// no relative property in the full-text constraint
final boolean nonFullTextConstraints = parent.isEmpty();
final int parentDepth = getDepth(parent);
QueryEngineSettings settings = filter.getQueryEngineSettings();
Iterator<LuceneResultRow> itr = new AbstractIterator<LuceneResultRow>() {
private final Deque<LuceneResultRow> queue = Queues.newArrayDeque();
private final Set<String> seenPaths = Sets.newHashSet();
private ScoreDoc lastDoc;
@Override
protected LuceneResultRow computeNext() {
while (!queue.isEmpty() || loadDocs()) {
return queue.remove();
}
return endOfData();
}
private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher) throws IOException {
IndexReader reader = searcher.getIndexReader();
PathStoredFieldVisitor visitor = new PathStoredFieldVisitor();
reader.document(doc.doc, visitor);
String path = visitor.getPath();
if (path != null) {
if ("".equals(path)) {
path = "/";
}
if (!parent.isEmpty()) {
// TODO OAK-828 this breaks node aggregation
// get the base path
// ensure the path ends with the given
// relative path
// if (!path.endsWith("/" + parent)) {
// continue;
// }
path = getAncestorPath(path, parentDepth);
// avoid duplicate entries
if (seenPaths.contains(path)) {
return null;
}
seenPaths.add(path);
}
return new LuceneResultRow(path, doc.score);
}
return null;
}
/**
* Loads the lucene documents in batches
* @return true if any document is loaded
*/
private boolean loadDocs() {
IndexNode indexNode = null;
IndexSearcher searcher = null;
ScoreDoc lastDocToRecord = null;
try {
indexNode = acquire();
searcher = indexNode.acquireSearcher();
Query query = getQuery(filter, searcher.getIndexReader(),
nonFullTextConstraints, analyzer, index.getDefinition());
TopDocs docs;
if (lastDoc != null) {
docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE);
} else {
docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE);
}
for (ScoreDoc doc : docs.scoreDocs) {
LuceneResultRow row = convertToRow(doc, searcher);
if(row != null) {
queue.add(row);
}
lastDocToRecord = doc;
}
} catch (IOException e) {
LOG.warn("query via {} failed.", LuceneIndex.this, e);
} finally {
release(indexNode, searcher);
}
if (lastDocToRecord != null) {
this.lastDoc = lastDocToRecord;
}
return !queue.isEmpty();
}
private IndexNode acquire() {
return tracker.getIndexNode("/");
}
private void release(IndexNode indexNode, IndexSearcher searcher){
try {
if(searcher != null){
indexNode.releaseSearcher();
}
} catch (IOException e) {
LOG.warn("Error occurred while releasing/closing the " +
"IndexSearcher", e);
}
}
};
return new LucenePathCursor(itr, settings);
}
/**
* Get the Lucene query for the given filter.
*
* @param filter the filter, including full-text constraint
* @param reader the Lucene reader
* @param nonFullTextConstraints whether non-full-text constraints (such a
* path, node type, and so on) should be added to the Lucene
* query
* @param analyzer the Lucene analyzer used for building the fulltext query
* @param indexDefinition nodestate that contains the index definition
* @return the Lucene query
*/
private static Query getQuery(Filter filter, IndexReader reader,
boolean nonFullTextConstraints, Analyzer analyzer, NodeState indexDefinition) {
List<Query> qs = new ArrayList<Query>();
FullTextExpression ft = filter.getFullTextConstraint();
if (ft == null) {
// there might be no full-text constraint
// when using the LowCostLuceneIndexProvider
// which is used for testing
} else {
qs.add(getFullTextQuery(ft, analyzer, reader));
}
PropertyRestriction pr = filter.getPropertyRestriction(NATIVE_QUERY_FUNCTION);
if (pr != null) {
String query = String.valueOf(pr.first.getValue(pr.first.getType()));
QueryParser queryParser = new QueryParser(VERSION, "", analyzer);
if (query.startsWith("mlt?")) {
String mltQueryString = query.replace("mlt?", "");
if (reader != null) {
Query moreLikeThis = MoreLikeThisHelper.getMoreLikeThis(reader, analyzer, mltQueryString);
if (moreLikeThis != null) {
qs.add(moreLikeThis);
}
}
}
else {
try {
qs.add(queryParser.parse(query));
} catch (ParseException e) {
throw new RuntimeException(e);
}
}
} else if (nonFullTextConstraints) {
addNonFullTextConstraints(qs, filter, reader, analyzer,
indexDefinition);
}
if (qs.size() == 0) {
return new MatchAllDocsQuery();
}
if (qs.size() == 1) {
return qs.get(0);
}
BooleanQuery bq = new BooleanQuery();
for (Query q : qs) {
bq.add(q, MUST);
}
return bq;
}
private static void addNonFullTextConstraints(List<Query> qs,
Filter filter, IndexReader reader, Analyzer analyzer, NodeState indexDefinition) {
if (!filter.matchesAllTypes()) {
addNodeTypeConstraints(qs, filter);
}
String path = filter.getPath();
switch (filter.getPathRestriction()) {
case ALL_CHILDREN:
if ("/".equals(path)) {
break;
}
if (!path.endsWith("/")) {
path += "/";
}
qs.add(new PrefixQuery(newPathTerm(path)));
break;
case DIRECT_CHILDREN:
if (!path.endsWith("/")) {
path += "/";
}
qs.add(new PrefixQuery(newPathTerm(path)));
break;
case EXACT:
qs.add(new TermQuery(newPathTerm(path)));
break;
case PARENT:
if (denotesRoot(path)) {
// there's no parent of the root node
// we add a path that can not possibly occur because there
// is no way to say "match no documents" in Lucene
qs.add(new TermQuery(new Term(FieldNames.PATH, "///")));
} else {
qs.add(new TermQuery(newPathTerm(getParentPath(path))));
}
break;
case NO_RESTRICTION:
break;
}
for (PropertyRestriction pr : filter.getPropertyRestrictions()) {
if (pr.first == null && pr.last == null) {
// ignore property existence checks, Lucene can't to 'property
// is not null' queries (OAK-1208)
continue;
}
// check excluded properties and types
if (isExcludedProperty(pr, indexDefinition)) {
continue;
}
String name = pr.propertyName;
if ("rep:excerpt".equals(name)) {
continue;
}
if (JCR_PRIMARYTYPE.equals(name)) {
continue;
}
if (skipTokenization(name)) {
qs.add(new TermQuery(new Term(name, pr.first
.getValue(STRING))));
continue;
}
String first = null;
String last = null;
boolean isLike = pr.isLike;
// TODO what to do with escaped tokens?
if (pr.first != null) {
first = pr.first.getValue(STRING);
first = first.replace("\\", "");
}
if (pr.last != null) {
last = pr.last.getValue(STRING);
last = last.replace("\\", "");
}
if (isLike) {
first = first.replace('%', WildcardQuery.WILDCARD_STRING);
first = first.replace('_', WildcardQuery.WILDCARD_CHAR);
int indexOfWS = first.indexOf(WildcardQuery.WILDCARD_STRING);
int indexOfWC = first.indexOf(WildcardQuery.WILDCARD_CHAR);
int len = first.length();
if (indexOfWS == len || indexOfWC == len) {
// remove trailing "*" for prefixquery
first = first.substring(0, first.length() - 1);
if (JCR_PATH.equals(name)) {
qs.add(new PrefixQuery(newPathTerm(first)));
} else {
qs.add(new PrefixQuery(new Term(name, first)));
}
} else {
if (JCR_PATH.equals(name)) {
qs.add(new WildcardQuery(newPathTerm(first)));
} else {
qs.add(new WildcardQuery(new Term(name, first)));
}
}
continue;
}
if (first != null && first.equals(last) && pr.firstIncluding
&& pr.lastIncluding) {
if (JCR_PATH.equals(name)) {
qs.add(new TermQuery(newPathTerm(first)));
} else {
if ("*".equals(name)) {
addReferenceConstraint(first, qs, reader);
} else {
for (String t : tokenize(first, analyzer)) {
qs.add(new TermQuery(new Term(name, t)));
}
}
}
continue;
}
first = tokenizeAndPoll(first, analyzer);
last = tokenizeAndPoll(last, analyzer);
qs.add(TermRangeQuery.newStringRange(name, first, last,
pr.firstIncluding, pr.lastIncluding));
}
}
private static String tokenizeAndPoll(String token, Analyzer analyzer){
if (token != null) {
List<String> tokens = tokenize(token, analyzer);
if (!tokens.isEmpty()) {
token = tokens.get(0);
}
}
return token;
}
private static boolean isExcludedProperty(PropertyRestriction pr,
NodeState definition) {
String name = pr.propertyName;
if (name.contains("/")) {
// lucene cannot handle child-level property restrictions
return true;
}
// check name
for (String e : definition.getStrings(EXCLUDE_PROPERTY_NAMES)) {
if (e.equalsIgnoreCase(name)) {
return true;
}
}
// check type
Integer type = null;
if (pr.first != null) {
type = pr.first.getType().tag();
} else if (pr.last != null) {
type = pr.last.getType().tag();
} else if (pr.list != null && !pr.list.isEmpty()) {
type = pr.list.get(0).getType().tag();
}
if (type != null) {
boolean isIn = false;
for (String e : definition.getStrings(INCLUDE_PROPERTY_TYPES)) {
if (PropertyType.valueFromName(e) == type) {
isIn = true;
}
}
if (!isIn) {
return true;
}
}
return false;
}
private static void addReferenceConstraint(String uuid, List<Query> qs,
IndexReader reader) {
if (reader == null) {
// getPlan call
qs.add(new TermQuery(new Term("*", uuid)));
return;
}
// reference query
BooleanQuery bq = new BooleanQuery();
Collection<String> fields = MultiFields.getIndexedFields(reader);
for (String f : fields) {
bq.add(new TermQuery(new Term(f, uuid)), SHOULD);
}
qs.add(bq);
}
private static void addNodeTypeConstraints(List<Query> qs, Filter filter) {
BooleanQuery bq = new BooleanQuery();
for (String type : filter.getPrimaryTypes()) {
bq.add(new TermQuery(new Term(JCR_PRIMARYTYPE, type)), SHOULD);
}
for (String type : filter.getMixinTypes()) {
bq.add(new TermQuery(new Term(JCR_MIXINTYPES, type)), SHOULD);
}
qs.add(bq);
}
static Query getFullTextQuery(FullTextExpression ft, final Analyzer analyzer, final IndexReader reader) {
// a reference to the query, so it can be set in the visitor
// (a "non-local return")
final AtomicReference<Query> result = new AtomicReference<Query>();
ft.accept(new FullTextVisitor() {
@Override
public boolean visit(FullTextOr or) {
BooleanQuery q = new BooleanQuery();
for (FullTextExpression e : or.list) {
Query x = getFullTextQuery(e, analyzer, reader);
q.add(x, SHOULD);
}
result.set(q);
return true;
}
@Override
public boolean visit(FullTextAnd and) {
BooleanQuery q = new BooleanQuery();
for (FullTextExpression e : and.list) {
Query x = getFullTextQuery(e, analyzer, reader);
// Lucene can't deal with "must(must_not(x))"
if (x instanceof BooleanQuery) {
BooleanQuery bq = (BooleanQuery) x;
for (BooleanClause c : bq.clauses()) {
q.add(c);
}
} else {
q.add(x, MUST);
}
}
result.set(q);
return true;
}
@Override
public boolean visit(FullTextTerm term) {
String p = term.getPropertyName();
if (p != null && p.indexOf('/') >= 0) {
p = getName(p);
}
Query q = tokenToQuery(term.getText(), p, analyzer, reader);
if (q == null) {
return false;
}
String boost = term.getBoost();
if (boost != null) {
q.setBoost(Float.parseFloat(boost));
}
if (term.isNot()) {
BooleanQuery bq = new BooleanQuery();
bq.add(q, MUST_NOT);
result.set(bq);
} else {
result.set(q);
}
return true;
}
});
return result.get();
}
static Query tokenToQuery(String text, String fieldName, Analyzer analyzer, IndexReader reader) {
if (analyzer == null) {
return null;
}
List<String> tokens = tokenize(text, analyzer);
if (tokens.isEmpty()) {
// TODO what should be returned in the case there are no tokens?
return new BooleanQuery();
}
if (tokens.size() == 1) {
String token = tokens.iterator().next();
if (hasFulltextToken(token)) {
return new WildcardQuery(newFulltextTerm(token, fieldName));
} else {
return new TermQuery(newFulltextTerm(token, fieldName));
}
} else {
if (hasFulltextToken(tokens)) {
MultiPhraseQuery mpq = new MultiPhraseQuery();
for(String token: tokens){
if (hasFulltextToken(token)) {
Term[] terms = extractMatchingTokens(reader, fieldName, token);
if (terms != null && terms.length > 0) {
mpq.add(terms);
}
} else {
mpq.add(newFulltextTerm(token, fieldName));
}
}
return mpq;
} else {
PhraseQuery pq = new PhraseQuery();
for (String t : tokens) {
pq.add(newFulltextTerm(t, fieldName));
}
return pq;
}
}
}
private static Term[] extractMatchingTokens(IndexReader reader, String fieldName, String token) {
if (reader == null) {
// getPlan call
return null;
}
try {
List<Term> terms = new ArrayList<Term>();
Term onTerm = newFulltextTerm(token, fieldName);
Terms t = MultiFields.getTerms(reader, onTerm.field());
Automaton a = WildcardQuery.toAutomaton(onTerm);
CompiledAutomaton ca = new CompiledAutomaton(a);
TermsEnum te = ca.getTermsEnum(t);
BytesRef text;
while ((text = te.next()) != null) {
terms.add(newFulltextTerm(text.utf8ToString(), fieldName));
}
return terms.toArray(new Term[terms.size()]);
} catch (IOException e) {
LOG.error("Building fulltext query failed", e.getMessage());
return null;
}
}
private static boolean hasFulltextToken(List<String> tokens) {
for (String token : tokens) {
if (hasFulltextToken(token)) {
return true;
}
}
return false;
}
private static boolean hasFulltextToken(String token) {
for (char c : fulltextTokens) {
if (token.indexOf(c) != -1) {
return true;
}
}
return false;
}
private static char[] fulltextTokens = new char[] { '*', '?' };
/**
* Tries to merge back tokens that are split on relevant fulltext query
* wildcards ('*' or '?')
*
*
* @param text
* @param analyzer
* @return
*/
static List<String> tokenize(String text, Analyzer analyzer) {
List<String> tokens = new ArrayList<String>();
TokenStream stream = null;
try {
stream = analyzer.tokenStream(FieldNames.FULLTEXT,
new StringReader(text));
CharTermAttribute termAtt = stream
.addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = stream
.addAttribute(OffsetAttribute.class);
// TypeAttribute type = stream.addAttribute(TypeAttribute.class);
stream.reset();
int poz = 0;
boolean hasFulltextToken = false;
StringBuilder token = new StringBuilder();
while (stream.incrementToken()) {
String term = termAtt.toString();
int start = offsetAtt.startOffset();
int end = offsetAtt.endOffset();
if (start > poz) {
for (int i = poz; i < start; i++) {
for (char c : fulltextTokens) {
if (c == text.charAt(i)) {
token.append(c);
hasFulltextToken = true;
}
}
}
}
poz = end;
if (hasFulltextToken) {
token.append(term);
hasFulltextToken = false;
} else {
if (token.length() > 0) {
tokens.add(token.toString());
}
token = new StringBuilder();
token.append(term);
}
}
// consume to the end of the string
if (poz < text.length()) {
for (int i = poz; i < text.length(); i++) {
for (char c : fulltextTokens) {
if (c == text.charAt(i)) {
token.append(c);
}
}
}
}
if (token.length() > 0) {
tokens.add(token.toString());
}
stream.end();
} catch (IOException e) {
LOG.error("Building fulltext query failed", e.getMessage());
return null;
} finally {
try {
if (stream != null) {
stream.close();
}
} catch (IOException e) {
// ignore
}
}
return tokens;
}
@Override
public NodeAggregator getNodeAggregator() {
return aggregator;
}
static class LuceneResultRow {
final String path;
final double score;
LuceneResultRow(String path, double score) {
this.path = path;
this.score = score;
}
@Override
public String toString() {
return String.format("%s (%1.2f)", path, score);
}
}
/**
* A cursor over Lucene results. The result includes the path,
* and the jcr:score pseudo-property as returned by Lucene.
*/
static class LucenePathCursor implements Cursor {
private final Cursor pathCursor;
LuceneResultRow currentRow;
LucenePathCursor(final Iterator<LuceneResultRow> it, QueryEngineSettings settings) {
Iterator<String> pathIterator = new Iterator<String>() {
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public String next() {
currentRow = it.next();
return currentRow.path;
}
@Override
public void remove() {
it.remove();
}
};
pathCursor = new PathCursor(pathIterator, true, settings);
}
@Override
public boolean hasNext() {
return pathCursor.hasNext();
}
@Override
public void remove() {
pathCursor.remove();
}
@Override
public IndexRow next() {
final IndexRow pathRow = pathCursor.next();
return new IndexRow() {
@Override
public String getPath() {
return pathRow.getPath();
}
@Override
public PropertyValue getValue(String columnName) {
// overlay the score
if (QueryImpl.JCR_SCORE.equals(columnName)) {
return PropertyValues.newDouble(currentRow.score);
}
return pathRow.getValue(columnName);
}
};
}
}
private static class PathStoredFieldVisitor extends StoredFieldVisitor {
private String path;
private boolean pathVisited;
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (PATH.equals(fieldInfo.name)) {
return Status.YES;
}
return pathVisited ? Status.STOP : Status.NO;
}
@Override
public void stringField(FieldInfo fieldInfo, String value)
throws IOException {
if (PATH.equals(fieldInfo.name)) {
path = value;
pathVisited = true;
}
}
public String getPath() {
return path;
}
}
}