log.info(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString());
}
// Create BatchScanner, set the ranges, and setup the iterators.
optimizedEventQuery.start();
BatchScanner bs = null;
try {
bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads);
bs.setRanges(ranges);
IteratorSetting si = new IteratorSetting(21, "eval", OptimizedQueryIterator.class);
if (log.isDebugEnabled()) {
log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString);
}
// Set the query option
si.addOption(EvaluatingIterator.QUERY_OPTION, queryString);
// Set the Indexed Terms List option. This is the field name and normalized field value pair separated
// by a comma.
StringBuilder buf = new StringBuilder();
String sep = "";
for (Entry<String,String> entry : termIndexInfo.getFieldNamesAndValues().entries()) {
buf.append(sep);
buf.append(entry.getKey());
buf.append(":");
buf.append(termIndexInfo.getIndexValuesToOriginalValues().get(entry.getValue()));
buf.append(":");
buf.append(entry.getValue());
if (sep.equals("")) {
sep = ";";
}
}
if (log.isDebugEnabled()) {
log.debug("Setting scan option: " + FieldIndexQueryReWriter.INDEXED_TERMS_LIST + " to " + buf.toString());
}
FieldIndexQueryReWriter rewriter = new FieldIndexQueryReWriter();
String q = "";
try {
q = queryString;
q = rewriter.applyCaseSensitivity(q, true, false);// Set upper/lower case for fieldname/fieldvalue
Map<String,String> opts = new HashMap<String,String>();
opts.put(FieldIndexQueryReWriter.INDEXED_TERMS_LIST, buf.toString());
q = rewriter.removeNonIndexedTermsAndInvalidRanges(q, opts);
q = rewriter.applyNormalizedTerms(q, opts);
if (log.isDebugEnabled()) {
log.debug("runServerQuery, FieldIndex Query: " + q);
}
} catch (org.apache.commons.jexl2.parser.ParseException ex) {
log.error("Could not parse query, Jexl ParseException: " + ex);
} catch (Exception ex) {
log.error("Problem rewriting query, Exception: " + ex.getMessage());
}
si.addOption(BooleanLogicIterator.FIELD_INDEX_QUERY, q);
// Set the term cardinality option
sep = "";
buf.delete(0, buf.length());
for (Entry<String,Long> entry : termIndexInfo.getTermCardinality().entrySet()) {
buf.append(sep);
buf.append(entry.getKey());
buf.append(":");
buf.append(entry.getValue());
sep = ",";
}
if (log.isDebugEnabled())
log.debug("Setting scan option: " + BooleanLogicIterator.TERM_CARDINALITIES + " to " + buf.toString());
si.addOption(BooleanLogicIterator.TERM_CARDINALITIES, buf.toString());
if (this.useReadAheadIterator) {
if (log.isDebugEnabled()) {
log.debug("Enabling read ahead iterator with queue size: " + this.readAheadQueueSize + " and timeout: " + this.readAheadTimeOut);
}
si.addOption(ReadAheadIterator.QUEUE_SIZE, this.readAheadQueueSize);
si.addOption(ReadAheadIterator.TIMEOUT, this.readAheadTimeOut);
}
if (null != unevaluatedExpressions) {
StringBuilder unevaluatedExpressionList = new StringBuilder();
String sep2 = "";
for (String exp : unevaluatedExpressions) {
unevaluatedExpressionList.append(sep2).append(exp);
sep2 = ",";
}
if (log.isDebugEnabled())
log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString());
si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString());
}
bs.addScanIterator(si);
processResults.start();
processResults.suspend();
long count = 0;
for (Entry<Key,Value> entry : bs) {
count++;
// The key that is returned by the EvaluatingIterator is not the same key that is in
// the table. The value that is returned by the EvaluatingIterator is a kryo
// serialized EventFields object.
processResults.resume();
Document d = this.createDocument(entry.getKey(), entry.getValue());
results.getResults().add(d);
processResults.suspend();
}
log.info(count + " matching entries found in optimized query.");
optimizationSucceeded = true;
processResults.stop();
} catch (TableNotFoundException e) {
log.error(this.getTableName() + "not found", e);
throw new RuntimeException(this.getIndexTableName() + "not found", e);
} finally {
if (bs != null) {
bs.close();
}
}
optimizedEventQuery.stop();
}
optimizedQuery.stop();
}
// WE should look into finding a better way to handle whether we do an optimized query or not.
// We are not setting up an else condition here because we may have aborted the logic early in the if statement.
if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()) && !orsAllIndexed)) {
// if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()))) {
fullScanQuery.start();
if (log.isDebugEnabled()) {
log.debug(hash + " Performing full scan query");
}
// Set up a full scan using the date ranges from the query
// Create BatchScanner, set the ranges, and setup the iterators.
BatchScanner bs = null;
try {
// The ranges are the start and end dates
Collection<Range> r = getFullScanRange(beginDate, endDate, terms);
ranges.addAll(r);
if (log.isDebugEnabled()) {
log.debug(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString());
}
bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads);
bs.setRanges(ranges);
IteratorSetting si = new IteratorSetting(22, "eval", EvaluatingIterator.class);
// Create datatype regex if needed
if (null != typeFilter) {
StringBuilder buf = new StringBuilder();
String s = "";
for (String type : typeFilter) {
buf.append(s).append(type).append(".*");
s = "|";
}
if (log.isDebugEnabled())
log.debug("Setting colf regex iterator to: " + buf.toString());
IteratorSetting ri = new IteratorSetting(21, "typeFilter", RegExFilter.class);
RegExFilter.setRegexs(ri, null, buf.toString(), null, null, false);
bs.addScanIterator(ri);
}
if (log.isDebugEnabled()) {
log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString);
}
si.addOption(EvaluatingIterator.QUERY_OPTION, queryString);
if (null != unevaluatedExpressions) {
StringBuilder unevaluatedExpressionList = new StringBuilder();
String sep2 = "";
for (String exp : unevaluatedExpressions) {
unevaluatedExpressionList.append(sep2).append(exp);
sep2 = ",";
}
if (log.isDebugEnabled())
log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString());
si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString());
}
bs.addScanIterator(si);
long count = 0;
processResults.start();
processResults.suspend();
for (Entry<Key,Value> entry : bs) {
count++;
// The key that is returned by the EvaluatingIterator is not the same key that is in
// the partition table. The value that is returned by the EvaluatingIterator is a kryo
// serialized EventFields object.
processResults.resume();
Document d = this.createDocument(entry.getKey(), entry.getValue());
results.getResults().add(d);
processResults.suspend();
}
processResults.stop();
log.info(count + " matching entries found in full scan query.");
} catch (TableNotFoundException e) {
log.error(this.getTableName() + "not found", e);
} finally {
if (bs != null) {
bs.close();
}
}
fullScanQuery.stop();
}