CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", "cacheonly"));
final servletProperties prop = new servletProperties();
prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
// get segment
Segment indexSegment = null;
if (post != null && post.containsKey("segment")) {
final String segmentName = post.get("segment");
if (sb.indexSegments.segmentExist(segmentName)) {
indexSegment = sb.indexSegments.segment(segmentName);
}
} else {
// take default segment
indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
}
final String EXT = header.get("EXT", "");
final boolean rss = EXT.equals("rss");
final boolean json = EXT.equals("json");
prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
prop.put("promoteSearchPageGreeting.homepage", sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, ""));
prop.put("promoteSearchPageGreeting.smallImage", sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
if (post == null || indexSegment == null || env == null || !searchAllowed) {
// we create empty entries for template strings
prop.put("searchagain", "0");
prop.put("former", "");
prop.put("count", "10");
prop.put("offset", "0");
prop.put("resource", "global");
prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*"));
prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", ""));
prop.put("tenant", (post == null) ? "" : post.get("tenant", ""));
prop.put("indexof", "off");
prop.put("constraint", "");
prop.put("cat", "href");
prop.put("depth", "0");
prop.put("search.verify", (post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));
prop.put("search.navigation", (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
prop.put("contentdom", "text");
prop.put("contentdomCheckText", "1");
prop.put("contentdomCheckAudio", "0");
prop.put("contentdomCheckVideo", "0");
prop.put("contentdomCheckImage", "0");
prop.put("contentdomCheckApp", "0");
prop.put("excluded", "0");
prop.put("results", "");
prop.put("resultTable", "0");
prop.put("num-results", searchAllowed ? "0" : "4");
prop.put("num-results_totalcount", 0);
prop.put("num-results_offset", 0);
prop.put("num-results_itemsPerPage", 10);
prop.put("geoinfo", "0");
prop.put("rss_queryenc", "");
prop.put("meanCount", 5);
return prop;
}
// check for JSONP
if (post.containsKey("callback")) {
final String jsonp = post.get("callback")+ "([";
prop.put("jsonp-start", jsonp);
prop.put("jsonp-end", "])");
} else {
prop.put("jsonp-start", "");
prop.put("jsonp-end", "");
}
// Adding CORS Access header for yacysearch.rss output
if (rss) {
final ResponseHeader outgoingHeader = new ResponseHeader();
outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");
prop.setOutgoingHeader(outgoingHeader);
}
// collect search attributes
final boolean newsearch =post.hasValue("query") && post.hasValue("former") && !post.get("query","").equalsIgnoreCase(post.get("former","")); //new search term
int itemsPerPage = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 1000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 500), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
int offset = (newsearch) ? 0 : post.getInt("startRecord", post.getInt("offset", 0));
final int newcount;
if ( authenticated && (newcount = post.getInt("count", 0)) > 0 ) {
sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, newcount);
} // set new default maximumRecords if search with "more options"
boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0;
final boolean indexof = (post != null && post.get("indexof","").equals("on"));
final String originalUrlMask;
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) { // option search all
originalUrlMask = ".*";
} else if (!newsearch && post.containsKey("urlmaskfilter")) {
originalUrlMask = post.get("urlmaskfilter", ".*");
} else {
originalUrlMask = ".*";
}
String prefermask = (post == null) ? "" : post.get("prefermaskfilter", "");
if (!prefermask.isEmpty() && prefermask.indexOf(".*") < 0) {
prefermask = ".*" + prefermask + ".*";
}
Bitfield constraint = (post != null && post.containsKey("constraint") && !post.get("constraint", "").isEmpty()) ? new Bitfield(4, post.get("constraint", "______")) : null;
if (indexof) {
constraint = new Bitfield(4);
constraint.set(Condenser.flag_cat_indexof, true);
}
// SEARCH
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) ||
sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true);
global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global search
final boolean clustersearch = sb.isRobinsonMode() &&
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
sb.getConfig("cluster.mode", "").equals("publiccluster"));
if (clustersearch) {
global = true;
} // switches search on, but search target is limited to cluster nodes
// increase search statistic counter
if (!global) {
// we count only searches on the local peer here, because global searches
// are counted on the target peer to preserve privacy of the searcher
if (authenticated) {
// local or authenticated search requests are counted separately
// because they are not part of a public available peer statistic
sb.searchQueriesRobinsonFromLocal++;
} else {
// robinson-searches from non-authenticated requests are public
// and may be part of the public available statistic
sb.searchQueriesRobinsonFromRemote++;
}
}
// find search domain
final ContentDomain contentdom = ContentDomain.contentdomParser(post == null ? "text" : post.get("contentdom", "text"));
// patch until better search profiles are available
if ((contentdom != ContentDomain.TEXT) && (itemsPerPage <= 32)) {
itemsPerPage = 64;
}
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) {
trackerHandles = new TreeSet<Long>();
}
boolean block = false;
if (Domains.matchesList(client, sb.networkBlacklist)) {
global = false;
if (snippetFetchStrategy != null) {
snippetFetchStrategy = null;
}
block = true;
Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search");
} else if (Domains.matchesList(client, sb.networkWhitelist)) {
Log.logInfo("LOCAL_SEARCH", "ACCESS CONTROL: WHITELISTED CLIENT FROM " + client + " gets no search restrictions");
} else if (!authenticated && !localhostAccess) {
// in case that we do a global search or we want to fetch snippets, we check for DoS cases
synchronized (trackerHandles) {
final int accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size();
final int accInOneMinute = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size();
final int accInTenMinutes = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size();
// protections against too strong YaCy network load, reduces remote search
if (global) {
if (accInTenMinutes >= 60 || accInOneMinute >= 6 || accInThreeSeconds >= 1) {
global = false;
Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed global search");
}
}
// protection against too many remote server snippet loads (protects traffic on server)
if (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline()) {
if (accInTenMinutes >= 20 || accInOneMinute >= 4 || accInThreeSeconds >= 1) {
snippetFetchStrategy = CacheStrategy.CACHEONLY;
Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed remote snippet loading");
}
}
// general load protection
if (accInTenMinutes >= 3000 || accInOneMinute >= 600 || accInThreeSeconds >= 60) {
block = true;
Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed search");
}
}
}
if ((!block) && (post == null || post.get("cat", "href").equals("href"))) {
String urlmask = null;
// check available memory and clean up if necessary
if (!MemoryControl.request(8000000L, false)) {
indexSegment.urlMetadata().clearCache();
SearchEventCache.cleanupEvents(true);
}
final RankingProfile ranking = sb.getRanking();
if (querystring.indexOf("/near") >= 0) {
querystring = querystring.replace("/near", "");
ranking.coeff_worddistance = RankingProfile.COEFF_MAX;
}
if (querystring.indexOf("/date") >= 0) {
querystring = querystring.replace("/date", "");
ranking.coeff_date = RankingProfile.COEFF_MAX;
}
if (querystring.indexOf("/http") >= 0) {
querystring = querystring.replace("/http", "");
urlmask = "http://.*";
}
if (querystring.indexOf("/https") >= 0) {
querystring = querystring.replace("/https", "");
urlmask = "https://.*";
}
if (querystring.indexOf("/ftp") >= 0) {
querystring = querystring.replace("/ftp", "");
urlmask = "ftp://.*";
}
if (querystring.indexOf("/smb") >= 0) {
querystring = querystring.replace("/smb", "");
urlmask = "smb://.*";
}
if (querystring.indexOf("/file") >= 0) {
querystring = querystring.replace("/file", "");
urlmask = "file://.*";
}
if (querystring.indexOf("/location") >= 0) {
querystring = querystring.replace("/location", "");
if (constraint == null) {
constraint = new Bitfield(4);
}
constraint.set(Condenser.flag_cat_haslocation, true);
}
final int lrp = querystring.indexOf("/language/");
String lr = "";
if (lrp >= 0) {
if (querystring.length() >= (lrp + 11)) {
lr = querystring.substring(lrp + 9, lrp + 11);
}
querystring = querystring.replace("/language/" + lr, "");
lr = lr.toLowerCase();
}
final int inurl = querystring.indexOf("inurl:");
if (inurl >= 0) {
int ftb = querystring.indexOf(' ', inurl);
if (ftb == -1) {
ftb = querystring.length();
}
final String urlstr = querystring.substring(inurl + 6, ftb);
querystring = querystring.replace("inurl:" + urlstr, "");
if (!urlstr.isEmpty()) {
urlmask = ".*" + urlstr + ".*";
}
}
final int filetype = querystring.indexOf("filetype:");
if (filetype >= 0) {
int ftb = querystring.indexOf(' ', filetype);
if (ftb == -1) {
ftb = querystring.length();
}
String ft = querystring.substring(filetype + 9, ftb);
querystring = querystring.replace("filetype:" + ft, "");
while (!ft.isEmpty() && ft.charAt(0) == '.') ft = ft.substring(1);
if (!ft.isEmpty()) {
if (urlmask == null) {
urlmask = ".*\\." + ft;
} else {
urlmask = urlmask + ".*\\." + ft;
}
}
}
String tenant = null;
if (post.containsKey("tenant")) {
tenant = post.get("tenant");
if (tenant != null && tenant.isEmpty()) {
tenant = null;
}
if (tenant != null) {
if (urlmask == null) {
urlmask = ".*" + tenant + ".*";
} else urlmask = ".*" + tenant + urlmask;
}
}
final int site = querystring.indexOf("site:");
String sitehash = null;
String sitehost = null;
if (site >= 0) {
int ftb = querystring.indexOf(' ', site);
if (ftb == -1) {
ftb = querystring.length();
}
sitehost = querystring.substring(site + 5, ftb);
querystring = querystring.replace("site:" + sitehost, "");
while (sitehost.length() > 0 && sitehost.charAt(0) == '.') {
sitehost = sitehost.substring(1);
}
while (sitehost.endsWith(".")) {
sitehost = sitehost.substring(0, sitehost.length() - 1);
}
sitehash = DigestURI.hosthash(sitehost);
}
final int heuristicScroogle = querystring.indexOf("heuristic:scroogle");
if (heuristicScroogle >= 0) {
querystring = querystring.replace("heuristic:scroogle", "");
}
final int heuristicBlekko = querystring.indexOf("heuristic:blekko");
if (heuristicBlekko >= 0) {
querystring = querystring.replace("heuristic:blekko", "");
}
final int authori = querystring.indexOf("author:");
String authorhash = null;
if (authori >= 0) {
// check if the author was given with single quotes or without
final boolean quotes = (querystring.charAt(authori + 7) == (char) 39);
String author;
if (quotes) {
int ftb = querystring.indexOf((char) 39, authori + 8);
if (ftb == -1) {
ftb = querystring.length() + 1;
}
author = querystring.substring(authori + 8, ftb);
querystring = querystring.replace("author:'" + author + "'", "");
} else {
int ftb = querystring.indexOf(' ', authori);
if (ftb == -1) {
ftb = querystring.length();
}
author = querystring.substring(authori + 7, ftb);
querystring = querystring.replace("author:" + author, "");
}
authorhash = ASCII.String(Word.word2hash(author));
}
final int tld = querystring.indexOf("tld:");
if (tld >= 0) {
int ftb = querystring.indexOf(' ', tld);
if (ftb == -1) {
ftb = querystring.length();
}
String domain = querystring.substring(tld + 4, ftb);
querystring = querystring.replace("tld:" + domain, "");
while (domain.length() > 0 && domain.charAt(0) == '.') {
domain = domain.substring(1);
}
if (domain.indexOf('.') < 0) {
domain = "\\." + domain;
} // is tld
if (domain.length() > 0) {
urlmask = "[a-zA-Z]*://[^/]*" + domain + "/.*" + ((urlmask != null) ? urlmask : "");
}
}
if (urlmask == null || urlmask.isEmpty()) {
urlmask = originalUrlMask;
} //if no urlmask was given
// read the language from the language-restrict option 'lr'
// if no one is given, use the user agent or the system language as default
String language = (post == null) ? lr : post.get("lr", lr);
if (language.startsWith("lang_")) {
language = language.substring(5);
}
if (!ISO639.exists(language)) {
// find out language of the user by reading of the user-agent string
String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE);
if (agent == null) {
agent = System.getProperty("user.language");
}
language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent);
if (language == null) {
language = "en";
}
}
// navigation
final String navigation = (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "");
// the query
final TreeSet<String>[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute
final int maxDistance = (querystring.indexOf('"') >= 0) ? query.length - 1 : Integer.MAX_VALUE;
// filter out stopwords
final SortedSet<String> filtered = SetTools.joinConstructive(query[0], Switchboard.stopwords);
if (!filtered.isEmpty()) {
SetTools.excludeDestructive(query[0], Switchboard.stopwords);
}
// if a minus-button was hit, remove a special reference first
if (post != null && post.containsKey("deleteref")) {
try {
if (!sb.verifyAuthentication(header, true)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// delete the index entry locally
final String delHash = post.get("deleteref", ""); // urlhash
indexSegment.termIndex().remove(Word.words2hashesHandles(query[0]), delHash.getBytes());
// make new news message with negative voting
if (!sb.isRobinsonMode()) {
final Map<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
} catch (final IOException e) {
Log.logException(e);
}
}
// if a plus-button was hit, create new voting message
if (post != null && post.containsKey("recommendref")) {
if (!sb.verifyAuthentication(header, true)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
final String recommendHash = post.get("recommendref", ""); // urlhash
final URIMetadataRow urlentry = indexSegment.urlMetadata().load(UTF8.getBytes(recommendHash));
if (urlentry != null) {
final URIMetadataRow.Components metadata = urlentry.metadata();
Document[] documents = null;
try {
documents = sb.loader.loadDocuments(sb.loader.request(metadata.url(), true, false), CacheStrategy.IFEXIST, 5000, Long.MAX_VALUE);
} catch (final IOException e) {
} catch (final Parser.Failure e) {
}
if (documents != null) {
// create a news message
final Map<String, String> map = new HashMap<String, String>();
map.put("url", metadata.url().toNormalform(false, true).replace(',', '|'));
map.put("title", metadata.dc_title().replace(',', ' '));
map.put("description", documents[0].dc_title().replace(',', ' '));
map.put("author", documents[0].dc_creator());
map.put("tags", documents[0].dc_subject(' '));
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map);
documents[0].close();
}
}
}
// prepare search properties
final boolean globalsearch = (global) && indexReceiveGranted;
// do the search
final HandleSet queryHashes = Word.words2hashesHandles(query[0]);
final Pattern snippetPattern = QueryParams.stringSearchPattern(originalquerystring);
// check filters
try {
Pattern.compile(urlmask);
} catch (final PatternSyntaxException ex) {
Log.logWarning("SEARCH", "Illegal URL mask, not a valid regex: " + urlmask);
prop.put("urlmaskerror", 1);
prop.putHTML("urlmaskerror_urlmask", urlmask);
urlmask = ".*";
}
try {
Pattern.compile(prefermask);
} catch (final PatternSyntaxException ex) {
Log.logWarning("SEARCH", "Illegal prefer mask, not a valid regex: " + prefermask);
prop.put("prefermaskerror", 1);
prop.putHTML("prefermaskerror_prefermask", prefermask);
prefermask = "";
}
final QueryParams theQuery = new QueryParams(
originalquerystring,
queryHashes,
Word.words2hashesHandles(query[1]),
Word.words2hashesHandles(query[2]),
snippetPattern,
tenant,
maxDistance,
prefermask,
contentdom,
language,
navigation,
snippetFetchStrategy,
itemsPerPage,
offset,
urlmask,
(clustersearch && globalsearch) ? QueryParams.SEARCHDOM_CLUSTERALL :
((globalsearch) ? QueryParams.SEARCHDOM_GLOBALDHT : QueryParams.SEARCHDOM_LOCAL),
20,
constraint,
true,
sitehash,
authorhash,
DigestURI.TLD_any_zone_filter,
client,
authenticated,
indexSegment,
ranking,
header.get(RequestHeader.USER_AGENT, ""),
sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);
// filter out words that appear in bluelist
theQuery.filterOut(Switchboard.blueList);
// log
Log.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + QueryParams.hashSet2hashString(theQuery.queryHashes) + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
yacyChannel.channels(yacyChannel.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
final long timestamp = System.currentTimeMillis();
// create a new search event
if (SearchEventCache.getEvent(theQuery.id(false)) == null) {
theQuery.setOffset(0); // in case that this is a new search, always start without a offset
offset = 0;
}
final SearchEvent theSearch = SearchEventCache.getEvent(
theQuery, sb.peers, sb.tables, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false, sb.loader,
(int) sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),
sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)),
(int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0),
(int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
if (offset == 0) {
if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) {
sb.heuristicSite(theSearch, sitehost);
}
if ((heuristicScroogle >= 0 || sb.getConfigBool("heuristic.scroogle", false)) && authenticated) {
sb.heuristicScroogle(theSearch);
}
if ((heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated) {
sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko");
}
}
// log
Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
"local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
"-local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
"-local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
"remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " +
(System.currentTimeMillis() - timestamp) + " ms");
// prepare search statistics
theQuery.resultcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = theSearch.result().getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.result().getSnippetComputationTime();
AccessTracker.add(AccessTracker.Location.local, theQuery);
// check suggestions
final int meanMax = (post != null) ? post.getInt("meanCount", 0) : 0;
prop.put("meanCount", meanMax);
if (meanMax > 0 && !json && !rss) {
final DidYouMean didYouMean = new DidYouMean(indexSegment.termIndex(), querystring);
final Iterator<String> meanIt = didYouMean.getSuggestions(100, 5).iterator();
int meanCount = 0;
String suggestion;
while( meanCount<meanMax && meanIt.hasNext()) {
suggestion = meanIt.next();