if (query.collapse != null && (query.collapse.length > 0)) {
collapser = new CollapseFieldFilter(query.collapse, parseFormat);
}
//CDXLine prev = null;
CDXLine line = null;
//boolean prevUrlAllowed = true;
FieldSplitFormat outputFields = null;
if (!authChecker.isAllCdxFieldAccessAllowed(authToken)) {
outputFields = this.authChecker.getPublicCdxFormat();
}
if (!query.fl.isEmpty()) {
if (outputFields == null) {
outputFields = parseFormat;
}
try {
outputFields = outputFields.createSubset(URLDecoder.decode(query.fl, "UTF-8"));
} catch (UnsupportedEncodingException e) {
}
} else if (outputFields != null) {
outputFields = parseFormat.createSubset(outputFields);
}
outputProcessor.begin();
int writeCount = 0;
long allCount = 0;
int writeLimit = query.limit;
while (cdx.hasNext() && ((writeLimit == 0) || (writeCount < writeLimit)) && (allCount < readLimit) && !responseWriter.isAborted()) {
String rawLine = cdx.next();
allCount++;
if (query.offset > 0) {
--query.offset;
continue;
}
// prev = line;
//line = new CDXLine(rawLine, parseFormat);
line = this.cdxLineFactory.createStandardCDXLine(rawLine, parseFormat);
//TODO: better way to handle this special case?
if (line.getMimeType().equals("alexa/dat")) {
continue;
}
// Additional access check, per capture
if (accessChecker != null) {
if (!accessChecker.includeCapture(line)) {
continue;
}
}
// if (!authChecker.isAllUrlAccessAllowed(authToken)) {
// if ((query.matchType != MatchType.exact) && ((prev == null) || !line.getUrlKey().equals(prev.getUrlKey()))) {
// prevUrlAllowed = authChecker.isUrlAllowed(line.getOriginalUrl(), authToken);
// }
//
// if (!prevUrlAllowed) {
// continue;
// }
// }
//
// if (!authChecker.isCaptureAllowed(line, authToken)) {
// continue;
// }
//
outputProcessor.trackLine(line);
// Timestamp Range Filtering
String timestamp = line.getTimestamp();
if (!query.from.isEmpty() && (timestamp.compareTo(query.from) < 0)) {
continue;
}
if (!query.to.isEmpty() && (timestamp.compareTo(query.to) > 0) && !timestamp.startsWith(query.to)) {
if (query.matchType == MatchType.exact) {
break;
} else {
continue;
}
}
// Check regex matcher if it exists
if ((filterMatcher != null) && !filterMatcher.include(line)) {
continue;
}
// Check collapser
if ((collapser != null) && !collapser.include(line)) {
continue;
}
// Filter to only include output fields
if (outputFields != null) {
line = new CDXLine(line, outputFields);
}
writeCount += outputProcessor.writeLine(line);
if (Thread.interrupted()) {
break;
}
}
if (query.showResumeKey && (line != null) && (writeLimit > 0) && (writeCount >= writeLimit)) {
StringBuilder sb = new StringBuilder();
sb.append(line.getUrlKey());
sb.append(' ');
sb.append(UrlSurtRangeComputer.incLastChar(line.getTimestamp()));
String resumeKey;
try {
resumeKey = URLEncoder.encode(sb.toString(), "UTF-8");
outputProcessor.writeResumeKey(resumeKey);
} catch (UnsupportedEncodingException e) {