/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.solr.client.solrj.request.DocumentAnalysisRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.AnalysisParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.XMLErrorLogger;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
/**
* An analysis handler that provides a breakdown of the analysis process of provided docuemnts. This handler expects a
* (single) content stream of the following format:
* <p/>
* <pre><code>
* <docs>
* <doc>
* <field name="id">1</field>
* <field name="name">The Name</field>
* <field name="text">The Text Value</field>
* <doc>
* <doc>...</doc>
* <doc>...</doc>
* ...
* </docs>
* </code></pre>
* <p/>
* <em><b>Note: Each document must contain a field which serves as the unique key. This key is used in the returned
* response to assoicate an analysis breakdown to the analyzed document.</b></em>
* <p/>
* <p/>
* <p/>
* Like the {@link org.apache.solr.handler.FieldAnalysisRequestHandler}, this handler also supports query analysis by
* sending either an "analysis.query" or "q" request paraemter that holds the query text to be analyzed. It also
* supports the "analysis.showmatch" parameter which when set to {@code true}, all field tokens that match the query
* tokens will be marked as a "match".
*
* @version $Id: DocumentAnalysisRequestHandler.java 1075090 2011-02-27 17:20:30Z uschindler $
* @since solr 1.4
*/
public class DocumentAnalysisRequestHandler extends AnalysisRequestHandlerBase {
public static final Logger log = LoggerFactory.getLogger(DocumentAnalysisRequestHandler.class);
private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
private static final float DEFAULT_BOOST = 1.0f;
private XMLInputFactory inputFactory;
@Override
public void init(NamedList args) {
super.init(args);
inputFactory = XMLInputFactory.newInstance();
try {
// The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
// XMLInputFactory, as that implementation tries to cache and reuse the
// XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
// prevents this.
// All other known open-source stax parsers (and the bea ref impl)
// have thread-safe factories.
inputFactory.setProperty("reuse-instance", Boolean.FALSE);
} catch (IllegalArgumentException ex) {
// Other implementations will likely throw this exception since "reuse-instance"
// isimplementation specific.
log.debug("Unable to set the 'reuse-instance' property for the input factory: " + inputFactory);
}
inputFactory.setXMLReporter(xmllog);
}
/**
* {@inheritDoc}
*/
@Override
protected NamedList doAnalysis(SolrQueryRequest req) throws Exception {
DocumentAnalysisRequest analysisRequest = resolveAnalysisRequest(req);
return handleAnalysisRequest(analysisRequest, req.getSchema());
}
@Override
public String getDescription() {
return "Provides a breakdown of the analysis process of provided documents";
}
@Override
public String getVersion() {
return "$Revision: 1075090 $";
}
@Override
public String getSourceId() {
return "$Id: DocumentAnalysisRequestHandler.java 1075090 2011-02-27 17:20:30Z uschindler $";
}
@Override
public String getSource() {
return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene_solr_3_5/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java $";
}
//================================================ Helper Methods ==================================================
/**
* Resolves the {@link DocumentAnalysisRequest} from the given solr request.
*
* @param req The solr request.
*
* @return The resolved document analysis request.
*
* @throws IOException Thrown when reading/parsing the content stream of the request fails.
* @throws XMLStreamException Thrown when reading/parsing the content stream of the request fails.
*/
DocumentAnalysisRequest resolveAnalysisRequest(SolrQueryRequest req) throws IOException, XMLStreamException {
DocumentAnalysisRequest request = new DocumentAnalysisRequest();
SolrParams params = req.getParams();
String query = params.get(AnalysisParams.QUERY, params.get(CommonParams.Q, null));
request.setQuery(query);
boolean showMatch = params.getBool(AnalysisParams.SHOW_MATCH, false);
request.setShowMatch(showMatch);
ContentStream stream = extractSingleContentStream(req);
InputStream is = null;
XMLStreamReader parser = null;
try {
is = stream.getStream();
final String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
parser = (charset == null) ?
inputFactory.createXMLStreamReader(is) : inputFactory.createXMLStreamReader(is, charset);
while (true) {
int event = parser.next();
switch (event) {
case XMLStreamConstants.END_DOCUMENT: {
parser.close();
return request;
}
case XMLStreamConstants.START_ELEMENT: {
String currTag = parser.getLocalName();
if ("doc".equals(currTag)) {
log.trace("Reading doc...");
SolrInputDocument document = readDocument(parser, req.getSchema());
request.addDocument(document);
}
break;
}
}
}
} finally {
if (parser != null) parser.close();
IOUtils.closeQuietly(is);
}
}
/**
* Handles the resolved {@link DocumentAnalysisRequest} and returns the analysis response as a named list.
*
* @param request The {@link DocumentAnalysisRequest} to be handled.
* @param schema The index schema.
*
* @return The analysis response as a named list.
*/
NamedList<Object> handleAnalysisRequest(DocumentAnalysisRequest request, IndexSchema schema) {
SchemaField uniqueKeyField = schema.getUniqueKeyField();
NamedList<Object> result = new SimpleOrderedMap<Object>();
for (SolrInputDocument document : request.getDocuments()) {
NamedList<NamedList> theTokens = new SimpleOrderedMap<NamedList>();
result.add(document.getFieldValue(uniqueKeyField.getName()).toString(), theTokens);
for (String name : document.getFieldNames()) {
// there's no point of providing analysis to unindexed fields.
SchemaField field = schema.getField(name);
if (!field.indexed()) {
continue;
}
NamedList<Object> fieldTokens = new SimpleOrderedMap<Object>();
theTokens.add(name, fieldTokens);
FieldType fieldType = schema.getFieldType(name);
final String queryValue = request.getQuery();
Set<String> termsToMatch;
try {
termsToMatch = (queryValue != null && request.isShowMatch())
? getQueryTokenSet(queryValue, fieldType.getQueryAnalyzer())
: Collections.<String>emptySet();
} catch (Exception e) {
// ignore analysis exceptions since we are applying arbitrary text to all fields
termsToMatch = Collections.<String>emptySet();
}
if (request.getQuery() != null) {
try {
AnalysisContext analysisContext = new AnalysisContext(fieldType, fieldType.getQueryAnalyzer(), Collections.<String>emptySet());
NamedList<List<NamedList>> tokens = analyzeValue(request.getQuery(), analysisContext);
fieldTokens.add("query", tokens);
} catch (Exception e) {
// ignore analysis exceptions since we are applying arbitrary text to all fields
}
}
Analyzer analyzer = fieldType.getAnalyzer();
AnalysisContext analysisContext = new AnalysisContext(fieldType, analyzer, termsToMatch);
Collection<Object> fieldValues = document.getFieldValues(name);
NamedList<NamedList<List<NamedList>>> indexTokens = new SimpleOrderedMap<NamedList<List<NamedList>>>();
for (Object fieldValue : fieldValues) {
NamedList<List<NamedList>> tokens = analyzeValue(fieldValue.toString(), analysisContext);
indexTokens.add(String.valueOf(fieldValue), tokens);
}
fieldTokens.add("index", indexTokens);
}
}
return result;
}
/**
* Reads the document from the given xml stream reader. The following document format is expected:
* <p/>
* <pre><code>
* <doc>
* <field name="id">1</field>
* <field name="name">The Name</field>
* <field name="text">The Text Value</field>
* </doc>
* </code></pre>
* <p/>
* <p/>
* <em>NOTE: each read document is expected to have at least one field which serves as the unique key.</em>
*
* @param reader The {@link XMLStreamReader} from which the document will be read.
* @param schema The index schema. The schema is used to validate that the read document has a unique key field.
*
* @return The read document.
*
* @throws XMLStreamException When reading of the document fails.
*/
SolrInputDocument readDocument(XMLStreamReader reader, IndexSchema schema) throws XMLStreamException {
SolrInputDocument doc = new SolrInputDocument();
String uniqueKeyField = schema.getUniqueKeyField().getName();
StringBuilder text = new StringBuilder();
String fieldName = null;
boolean hasId = false;
while (true) {
int event = reader.next();
switch (event) {
// Add everything to the text
case XMLStreamConstants.SPACE:
case XMLStreamConstants.CDATA:
case XMLStreamConstants.CHARACTERS:
text.append(reader.getText());
break;
case XMLStreamConstants.END_ELEMENT:
if ("doc".equals(reader.getLocalName())) {
if (!hasId) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"All documents must contain a unique key value: '" + doc.toString() + "'");
}
return doc;
} else if ("field".equals(reader.getLocalName())) {
doc.addField(fieldName, text.toString(), DEFAULT_BOOST);
if (uniqueKeyField.equals(fieldName)) {
hasId = true;
}
}
break;
case XMLStreamConstants.START_ELEMENT:
text.setLength(0);
String localName = reader.getLocalName();
if (!"field".equals(localName)) {
log.warn("unexpected XML tag doc/" + localName);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unexpected XML tag doc/" + localName);
}
for (int i = 0; i < reader.getAttributeCount(); i++) {
String attrName = reader.getAttributeLocalName(i);
if ("name".equals(attrName)) {
fieldName = reader.getAttributeValue(i);
}
}
break;
}
}
}
/**
* Extracts the only content stream from the request. {@link org.apache.solr.common.SolrException.ErrorCode#BAD_REQUEST}
* error is thrown if the request doesn't hold any content stream or holds more than one.
*
* @param req The solr request.
*
* @return The single content stream which holds the documents to be analyzed.
*/
private ContentStream extractSingleContentStream(SolrQueryRequest req) {
Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"DocumentAnlysisRequestHandler expects a single content stream with documents to analyze");
}
Iterator<ContentStream> iter = streams.iterator();
if (!iter.hasNext()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"DocumentAnlysisRequestHandler expects a single content stream with documents to analyze");
}
ContentStream stream = iter.next();
if (iter.hasNext()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"DocumentAnlysisRequestHandler expects a single content stream with documents to analyze");
}
return stream;
}
}