Examples of de.anomic.server.servletProperties

de.anomic.server.servletProperties

                else
                    serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));


                final ResponseHeader responseHeader = getDefaultHeaders(path);
                responseHeader.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"admin log-in\"");
                final servletProperties tp=new servletProperties();
                tp.put("returnto", path);
                HTTPDemon.sendRespondError(conProp, out, 5, 401, "Wrong Authentication", "", new File("proxymsg/authfail.inc"), tp, null, responseHeader);
                return;
            }


            // Authentication successful. remove brute-force flag
            serverCore.bfHost.remove(conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));


            // parse arguments
            serverObjects args = new serverObjects();
            int argc = 0;
            if (argsString == null) {
                // no args here, maybe a POST with multipart extension
                final int length = requestHeader.getContentLength();
                //System.out.println("HEADER: " + requestHeader.toString()); // DEBUG


                /* don't parse body in case of a POST CGI call since it has to be
                 * handed over to the CGI script unaltered and parsed by the script
                 */
                if (method.equals(HeaderFramework.METHOD_POST) &&
                        !(switchboard.getConfigBool("cgi.allow", false) &&
                        matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)))
                        ) {


                    // if its a POST, it can be either multipart or as args in the body
                    if ((requestHeader.containsKey(HeaderFramework.CONTENT_TYPE)) &&
                            (requestHeader.get(HeaderFramework.CONTENT_TYPE).toLowerCase().startsWith("multipart"))) {
                        // parse multipart
                        final Map<String, byte[]> files = HTTPDemon.parseMultipart(requestHeader, args, body);
                        // integrate these files into the args
                        if (files != null) {
                            final Iterator<Map.Entry<String, byte[]>> fit = files.entrySet().iterator();
                            Map.Entry<String, byte[]> entry;
                            while (fit.hasNext()) {
                                entry = fit.next();
                                args.put(entry.getKey() + "$file", entry.getValue());
                            }
                        }
                        argc = Integer.parseInt(requestHeader.get("ARGC"));
                    } else {
                        // parse args in body
                        argc = HTTPDemon.parseArgs(args, body, length);
                    }
                } else {
                    // no args
                    argsString = null;
                    args = null;
                    argc = 0;
                }
            } else {
                // simple args in URL (stuff after the "?")
                argc = HTTPDemon.parseArgs(args, argsString);
            }


            // check for cross site scripting - attacks in request arguments
            if (args != null && argc > 0) {
                // check all values for occurrences of script values
                final Iterator<String> e = args.values().iterator(); // enumeration of values
                String val;
                while (e.hasNext()) {
                    val = e.next();
                    if ((val != null) && (val.indexOf("<script") >= 0) && !path.equals("/Crawler_p.html")) {
                        // deny request
                        HTTPDemon.sendRespondError(conProp,out,4,403,null,"bad post values",null);
                        return;
                    }
                }
            }


            if (args != null) nocache = true;


            // we are finished with parsing
            // the result of value hand-over is in args and argc
            if (path.length() == 0) {
                HTTPDemon.sendRespondError(conProp,out,4,400,null,"Bad Request",null);
                out.flush();
                return;
            }
            File targetClass=null;


            // locate the file
            if (path.length() > 0 && path.charAt(0) != '/' && path.charAt(0) != '\\') path = "/" + path; // attach leading slash
            if (path.endsWith("index.html")) path = path.substring(0, path.length() - 10);


            // a different language can be desired (by i.e. ConfigBasic.html) than the one stored in the locale.language
            String localeSelection = switchboard.getConfig("locale.language","default");
            if (args != null && (args.containsKey("language"))) {
                // TODO 9.11.06 Bost: a class with information about available languages is needed.
                // the indexOf(".") is just a workaround because there from ConfigLanguage.html commes "de.lng" and
                // from ConfigBasic.html comes just "de" in the "language" parameter
                localeSelection = args.get("language", localeSelection);
                if (localeSelection.indexOf('.') != -1)
                    localeSelection = localeSelection.substring(0, localeSelection.indexOf('.'));
            }


            File targetFile = getLocalizedFile(path, localeSelection);
            String targetExt = (String) conProp.get("EXT"); if (targetExt == null) targetExt = "";
            targetClass = rewriteClassFile(new File(htDefaultPath, path));
            if (path.endsWith("/") || path.endsWith("\\")) {
                String testpath;
                // look for indexForward setting
                if (indexForward.length() > 0 && (targetFile = getOverlayedFile(path + indexForward)).exists()) {
                    testpath = path + indexForward;
                    targetClass = getOverlayedClass(testpath);
                    path = testpath;
                } else {
                    // attach default file name(s)
                    for (final String defaultFile : defaultFiles) {
                        testpath = path + defaultFile;
                        targetFile = getOverlayedFile(testpath);
                        targetClass = getOverlayedClass(testpath);
                        if (targetFile.exists()) {
                            path = testpath;
                            break;
                        }
                    }
                }
                targetFile = getLocalizedFile(path, localeSelection);


                //no defaultfile, send a dirlisting
                if (targetFile == null || !targetFile.exists() || (targetFile.exists() && targetFile.isDirectory())) {
                    final StringBuilder aBuffer = new StringBuilder();
                    aBuffer.append("<html>\n<head>\n</head>\n<body>\n<h1>Index of " + path + "</h1>\n  <ul>\n");
                    String[] list = targetFile.list();
                    if (list == null) list = new String[0]; // should not occur!
                    File f;
                    String size;
                    long sz;
                    String headline, author, description, publisher;
                    int images, links;
                    ContentScraper scraper;
                    for (final String element : list) {
                        f = new File(targetFile, element);
                        if (f.isDirectory()) {
                            aBuffer.append("    <li><a href=\"" + path + element + "/\">" + element + "/</a><br/></li>\n");
                        } else {
                            if (element.endsWith("html") || (element.endsWith("htm"))) {
                                scraper = ContentScraper.parseResource(f);
                                headline = scraper.getTitle();
                                author = scraper.getAuthor();
                                publisher = scraper.getPublisher();
                                description = scraper.getDescription();
                                images = scraper.getImages().size();
                                links = scraper.getAnchors().size();
                            } else {
                                headline = null;
                                author = null;
                                publisher = null;
                                description = null;
                                images = 0;
                                links = 0;
                            }
                            sz = f.length();
                            if (sz < 1024) {
                                size = sz + " bytes";
                            } else if (sz < 1024 * 1024) {
                                size = (sz / 1024) + " KB";
                            } else {
                                size = (sz / 1024 / 1024) + " MB";
                            }
                            aBuffer.append("    <li>");
                            if (headline != null && headline.length() > 0) aBuffer.append("<a href=\"" + element + "\"><b>" + headline + "</b></a><br/>");
                            aBuffer.append("<a href=\"" + path + element + "\">" + element + "</a><br/>");
                            if (author != null && author.length() > 0) aBuffer.append("Author: " + author + "<br/>");
                            if (publisher != null && publisher.length() > 0) aBuffer.append("Publisher: " + publisher + "<br/>");
                            if (description != null && description.length() > 0) aBuffer.append("Description: " + description + "<br/>");
                            aBuffer.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n");
                        }
                    }
                    aBuffer.append("  </ul>\n</body>\n</html>\n");


                    // write the list to the client
                    HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, "text/html; charset=UTF-8", aBuffer.length(), new Date(targetFile.lastModified()), null, new ResponseHeader(), null, null, true);
                    if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                        out.write(UTF8.getBytes(aBuffer.toString()));
                    }
                    return;
                }
            } else {
                    //XXX: you cannot share a .png/.gif file with a name like a class in htroot.
                    if ( !(targetFile.exists()) &&
                            !((path.endsWith("png")||path.endsWith("gif") ||
                            matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) ||
                            path.endsWith(".stream")) &&
                            targetClass!=null ) ){
                        targetFile = new File(htDocsPath, path);
                        targetClass = rewriteClassFile(new File(htDocsPath, path));
                    }
            }


            // implement proxy via url (not in servlet, because we need binary access on ouputStream)
            if (path.equals("/proxy.html")) {
              final List<Pattern> urlProxyAccess = Domains.makePatterns(sb.getConfig("proxyURL.access", "127.0.0.1"));
              if (sb.getConfigBool("proxyURL", false) && Domains.matchesList(clientIP, urlProxyAccess)) {
                doURLProxy(args, conProp, requestHeader, out);
                return;
              }
              else {
              HTTPDemon.sendRespondError(conProp,out,3,403,"Access denied",null,null);
              }
            }


            // track all files that had been accessed so far
            if (targetFile != null && targetFile.exists()) {
                if (args != null && args.size() > 0) sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), path));
            }


            //File targetClass = rewriteClassFile(targetFile);
            //We need tp here
            servletProperties templatePatterns = null;
            Date targetDate;


            if ((targetClass != null) && (path.endsWith("png"))) {
                // call an image-servlet to produce an on-the-fly - generated image
                Object img = null;
                try {
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "png");
                    // in case that there are no args given, args = null or empty hashmap
                    img = invokeServlet(targetClass, requestHeader, args);
                } catch (final InvocationTargetException e) {
                    theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
                    e.getMessage() +
                    " target exception at " + targetClass + ": " +
                    e.getTargetException().toString() + ":" +
                    e.getTargetException().getMessage() +
                    "; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
                    Log.logException(e);
                    Log.logException(e.getTargetException());
                    targetClass = null;
                }
                if (img == null) {
                    // error with image generation; send file-not-found
                    HTTPDemon.sendRespondError(conProp, out, 3, 404, "File not Found", null, null);
                } else {
                    if (img instanceof RasterPlotter) {
                        final RasterPlotter yp = (RasterPlotter) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
                        final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    if (img instanceof EncodedImage) {
                        final EncodedImage yp = (EncodedImage) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
                        final ByteBuffer result = yp.getImage();


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    /*
                    if (img instanceof BufferedImage) {
                        final BufferedImage i = (BufferedImage) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = MimeTable.ext2mime(targetExt, "text/html");


                        // generate an byte array from the generated image
                        int width = i.getWidth(); if (width < 0) width = 96; // bad hack
                        int height = i.getHeight(); if (height < 0) height = 96; // bad hack
                        final ByteBuffer result = RasterPlotter.exportImage(i, targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    */
                    if (img instanceof Image) {
                        final Image i = (Image) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = MimeTable.ext2mime(targetExt, "text/html");


                        // generate an byte array from the generated image
                        int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
                        int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
                        final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
                        bi.createGraphics().drawImage(i, 0, 0, width, height, null);
                        final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                }
            } else if (((switchboard.getConfigBool("cgi.allow", false)) &&                                  // check if CGI execution is allowed in config
                    (matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null))) &&  // "right" file extension?
                    (path.substring(0, path.indexOf(targetFile.getName())).contains("/CGI-BIN/") ||
                    path.substring(0, path.indexOf(targetFile.getName())).contains("/cgi-bin/")) &&         // file in right directory?
                    targetFile.exists())
                    ) {


                String mimeType = "text/html";
                int statusCode = 200;


                ProcessBuilder pb;


                pb = new ProcessBuilder(targetFile.getAbsolutePath());


                final String fileSeparator = System.getProperty("file.separator", "/");


                // set environment variables
                final Map<String, String> env = pb.environment();
                env.put("SERVER_SOFTWARE", getDefaultHeaders(path).get(HeaderFramework.SERVER));
                env.put("SERVER_NAME", sb.peers.mySeed().getName());
                env.put("GATEWAY_INTERFACE", "CGI/1.1");
                if (httpVersion != null) {
                    env.put("SERVER_PROTOCOL", httpVersion);
                }
                env.put("SERVER_PORT", switchboard.getConfig("port", "8090"));
                env.put("REQUEST_METHOD", method);
//                env.put("PATH_INFO", "");         // TODO: implement
//                env.put("PATH_TRANSLATED", "");   // TODO: implement
                env.put("SCRIPT_NAME", path);
                if (argsString != null) {
                    env.put("QUERY_STRING", argsString);
                }
                env.put("REMOTE_ADDR", clientIP);
//                env.put("AUTH_TYPE", "");         // TODO: implement
//                env.put("REMOTE_USER", "");       // TODO: implement
//                env.put("REMOTE_IDENT", "");      // I don't think we need this
                env.put("DOCUMENT_ROOT", switchboard.getAppPath().getAbsolutePath() + fileSeparator + switchboard.getConfig("htDocsPath", "DATA/HTDOCS"));
                if (requestHeader.getContentType() != null) {
                    env.put("CONTENT_TYPE", requestHeader.getContentType());
                }
                if (method.equalsIgnoreCase(HeaderFramework.METHOD_POST) && body != null) {
                    env.put("CONTENT_LENGTH", Integer.toString(requestHeader.getContentLength()));
                }


                // add values from request header to environment (see: http://hoohoo.ncsa.uiuc.edu/cgi/env.html#headers)
                for (final Map.Entry<String, String> requestHeaderEntry : requestHeader.entrySet()) {
                    env.put("HTTP_" + requestHeaderEntry.getKey().toUpperCase().replace("-", "_"), requestHeaderEntry.getValue());
                }


                int exitValue = 0;
                String cgiBody = null;


                try {
                    // start execution of script
                    final Process p = pb.start();


                    final OutputStream os = new BufferedOutputStream(p.getOutputStream());


                    if (method.equalsIgnoreCase(HeaderFramework.METHOD_POST) && body != null) {
                        final byte[] buffer = new byte[1024];
                        int len = requestHeader.getContentLength();
                        while (len > 0) {
                            body.read(buffer);
                            len = len - buffer.length;
                            os.write(buffer);
                        }
                    }


                    os.close();


                    try {
                        p.waitFor();
                    } catch (final InterruptedException ex) {


                    }


                    exitValue = p.exitValue();


                    final InputStream is = new BufferedInputStream(p.getInputStream());


                    final StringBuilder StringBuilder = new StringBuilder(1024);


                    while (is.available() > 0) {
                        StringBuilder.append((char) is.read());
                    }


                    final String cgiReturn = StringBuilder.toString();
                    int indexOfDelimiter = cgiReturn.indexOf("\n\n");
                    String[] cgiHeader = new String[0];
                    if (indexOfDelimiter > -1) {
                        cgiHeader = cgiReturn.substring(0, indexOfDelimiter).split("\n");
                    }
                    cgiBody = cgiReturn.substring(indexOfDelimiter + 1);


                    String key;
                    String value;
                    for (final String element : cgiHeader) {
                        indexOfDelimiter = element.indexOf(':');
                        key = element.substring(0, indexOfDelimiter).trim();
                        value = element.substring(indexOfDelimiter + 1).trim();
                        conProp.put(key, value);
                        if (key.equals("Cache-Control") && value.equals("no-cache")) {
                            nocache = true;
                        } else if (key.equals("Content-type")) {
                            mimeType = value;
                        } else if (key.equals("Status")) {
                            if (key.length() > 2) {
                                try {
                                    statusCode = Integer.parseInt(value.substring(0, 3));
                                } catch (final NumberFormatException ex) {
                                    /* tough luck, we will just have to use 200 as default value */
                                }
                            }
                        }
                    }
                } catch (final IOException ex) {
                    exitValue = -1;
                }


                /* did the script return an exit value != 0 and still there is supposed to be
                 * everything right with the HTTP status? -> change status to 500 since 200 would
                 * be a lie
                 */
                if (exitValue != 0 && statusCode == 200) {
                    statusCode = 500;
                }


                targetDate = new Date(System.currentTimeMillis());


                if (exitValue == 0 || (cgiBody != null && !cgiBody.equals(""))) {
                    HTTPDemon.sendRespondHeader(conProp, out, httpVersion, statusCode, null, mimeType, cgiBody.length(), targetDate, null, null, null, null, nocache);
                    out.write(UTF8.getBytes(cgiBody));
                } else {
                    HTTPDemon.sendRespondError(conProp, out, exitValue, statusCode, null, HeaderFramework.http1_1.get(Integer.toString(statusCode)), null);
                }




            } else if ((targetClass != null) && (path.endsWith(".stream"))) {
                // call rewrite-class
                requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "stream");
                //requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
                //requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);


                HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null);


                // in case that there are no args given, args = null or empty hashmap
                /* servletProperties tp = (servlerObjects) */ invokeServlet(targetClass, requestHeader, args);
                forceConnectionClose(conProp);
                return;
            } else if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
                // we have found a file that can be written to the client
                // if this file uses templates, then we use the template
                // re-write - method to create an result
                String mimeType = MimeTable.ext2mime(targetExt, "text/html");
                String ext = (String) conProp.get("EXT"); if (ext == null) ext = "";
                final boolean zipContent = requestHeader.acceptGzip() && HTTPDemon.shallTransportZipped("." + ext);
                if (path.endsWith("html") ||
                        path.endsWith("htm") ||
                        path.endsWith("xml") ||
                        path.endsWith("json") ||
                        path.endsWith("rdf") ||
                        path.endsWith("rss") ||
                        path.endsWith("csv") ||
                        path.endsWith("pac") ||
                        path.endsWith("src") ||
                        path.endsWith("vcf") ||
                        path.endsWith("kml") ||
                        path.endsWith("gpx") ||
                        path.endsWith("css") ||
                        path.endsWith("/") ||
                        path.equals("/robots.txt")) {


                    /*targetFile = getLocalizedFile(path);
                    if (!(targetFile.exists())) {
                        // try to find that file in the htDocsPath
                        File trialFile = new File(htDocsPath, path);
                        if (trialFile.exists()) targetFile = trialFile;
                    }*/




                    // call rewrite-class


                    if (targetClass != null) {
                        // CGI-class: call the class to create a property for rewriting
                        try {
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                            final int ep = path.lastIndexOf(".");
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, path.substring(ep + 1));
                            // in case that there are no args given, args = null or empty hashmap
                            final Object tmp = invokeServlet(targetClass, requestHeader, args);
                            if (tmp == null) {
                                // if no args given, then tp will be an empty Hashtable object (not null)
                                templatePatterns = new servletProperties();
                            } else if (tmp instanceof servletProperties) {
                                templatePatterns = (servletProperties) tmp;
                            } else {
                                templatePatterns = new servletProperties((serverObjects) tmp);
                            }
                            // check if the servlets requests authentication
                            if (templatePatterns.containsKey(servletProperties.ACTION_AUTHENTICATE)) {
                                // handle brute-force protection
                                if (realmProp != null) {
                                    Log.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
                                    final Integer attempts = serverCore.bfHost.get(clientIP);
                                    if (attempts == null)
                                        serverCore.bfHost.put(clientIP, Integer.valueOf(1));
                                    else
                                        serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
                                }
                                // send authentication request to browser
                                final ResponseHeader headers = getDefaultHeaders(path);
                                headers.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"" + templatePatterns.get(servletProperties.ACTION_AUTHENTICATE, "") + "\"");
                                HTTPDemon.sendRespondHeader(conProp,out,httpVersion,401,headers);
                                return;
                            } else if (templatePatterns.containsKey(servletProperties.ACTION_LOCATION)) {
                                String location = templatePatterns.get(servletProperties.ACTION_LOCATION, "");
                                if (location.length() == 0) location = path;


                                final ResponseHeader headers = getDefaultHeaders(path);
                                headers.setAdditionalHeaderProperties(templatePatterns.getOutgoingHeader().getAdditionalHeaderProperties()); //put the cookies into the new header TODO: can we put all headerlines, without trouble?
                                headers.put(HeaderFramework.LOCATION,location);
                                HTTPDemon.sendRespondHeader(conProp,out,httpVersion,302,headers);
                                return;
                            }
                            // add the application version, the uptime and the client name to every rewrite table
                            templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
                            templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() -  serverCore.startupTime) / 1000) / 60); // uptime in minutes
                            templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
                            templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID());
                            templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
                            final yacySeed myPeer = sb.peers.mySeed();
                            templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
                            templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
                            //System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug
                        } catch (final InvocationTargetException e) {
                            if (e.getCause() instanceof InterruptedException) {
                                throw new InterruptedException(e.getCause().getMessage());
                            }


                            theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
                                    e.getMessage() +
                                    " target exception at " + targetClass + ": " +
                                    e.getTargetException().toString() + ":" +
                                    e.getTargetException().getMessage(),e);
                            targetClass = null;
                            throw e;
                        }
                        nocache = true;
                    }


                    targetDate = new Date(targetFile.lastModified());


                    // rewrite the file
                    InputStream fis = null;


                    // read the file/template
                    TemplateCacheEntry templateCacheEntry = null;
                    final long fileSize = targetFile.length();
                    if (useTemplateCache && fileSize <= 512 * 1024) {
                        // read from cache
                        SoftReference<TemplateCacheEntry> ref = templateCache.get(targetFile);
                        if (ref != null) {
                            templateCacheEntry = ref.get();
                            if (templateCacheEntry == null) templateCache.remove(targetFile);
                        }


                        final Date targetFileDate = new Date(targetFile.lastModified());
                        if (templateCacheEntry == null || targetFileDate.after(templateCacheEntry.lastModified)) {
                            // loading the content of the template file into
                            // a byte array
                        templateCacheEntry = new TemplateCacheEntry();
                            templateCacheEntry.lastModified = targetFileDate;
                            templateCacheEntry.content = FileUtils.read(targetFile);


                            // storing the content into the cache
                            ref = new SoftReference<TemplateCacheEntry>(templateCacheEntry);
                            if (MemoryControl.shortStatus()) templateCache.clear();
                            templateCache.put(targetFile, ref);
                            if (theLogger.isFinest()) theLogger.logFinest("Cache MISS for file " + targetFile);
                        } else {
                            if (theLogger.isFinest()) theLogger.logFinest("Cache HIT for file " + targetFile);
                        }


                        // creating an inputstream needed by the template
                        // rewrite function
                        fis = new ByteArrayInputStream(templateCacheEntry.content);
                        templateCacheEntry = null;
                    } else if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
                        // read file completely into ram, avoid that too many files are open at the same time
                        fis = new ByteArrayInputStream(FileUtils.read(targetFile));
                    } else {
                        fis = new BufferedInputStream(new FileInputStream(targetFile));
                    }


                    if (mimeType.startsWith("text")) {
                        // every text-file distributed by yacy is UTF-8
                        if(!path.startsWith("/repository")) {
                            mimeType = mimeType + "; charset=UTF-8";
                        } else {
                            // detect charset of html-files
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
                        }
                    }


                    // write the array to the client
                    // we can do that either in standard mode (whole thing completely) or in chunked mode
                    // since yacy clients do not understand chunked mode (yet), we use this only for communication with the administrator
                    final boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
                    final boolean chunked = !method.equals(HeaderFramework.METHOD_HEAD) && !yacyClient && httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1);
                    if (chunked) {
                        // send page in chunks and parse SSIs
                        final ByteBuffer o = new ByteBuffer();
                        // apply templates
                        TemplateEngine.writeTemplate(fis, o, templatePatterns, ASCII.getBytes("-UNRESOLVED_PATTERN-"));
                        fis.close();
                        HTTPDemon.sendRespondHeader(conProp, out,
                                httpVersion, 200, null, mimeType, -1,
                                targetDate, null, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                null, "chunked", nocache);
                        // send the content in chunked parts, see RFC 2616 section 3.6.1
                        final ChunkedOutputStream chos = new ChunkedOutputStream(out);
                        ServerSideIncludes.writeSSI(o, chos, realmProp, clientIP);
                        //chos.write(result);
                        chos.finish();
                    } else {
                        // send page as whole thing, SSIs are not possible
                        final String contentEncoding = (zipContent) ? "gzip" : null;
                        // apply templates
                        final ByteBuffer o1 = new ByteBuffer();
                        TemplateEngine.writeTemplate(fis, o1, templatePatterns, ASCII.getBytes("-UNRESOLVED_PATTERN-"));
                        fis.close();
                        final ByteBuffer o = new ByteBuffer();


                        if (zipContent) {
                            GZIPOutputStream zippedOut = new GZIPOutputStream(o);
                            ServerSideIncludes.writeSSI(o1, zippedOut, realmProp, clientIP);
                            //httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
                            zippedOut.finish();
                            zippedOut.flush();
                            zippedOut.close();
                            zippedOut = null;
                        } else {
                            ServerSideIncludes.writeSSI(o1, o, realmProp, clientIP);
                            //httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
                        }
                        if (method.equals(HeaderFramework.METHOD_HEAD)) {
                            HTTPDemon.sendRespondHeader(conProp, out,
                                    httpVersion, 200, null, mimeType, o.length(),
                                    targetDate, null, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                    contentEncoding, null, nocache);
                        } else {
                            final byte[] result = o.getBytes(); // this interrupts streaming (bad idea!)
                            HTTPDemon.sendRespondHeader(conProp, out,
                                    httpVersion, 200, null, mimeType, result.length,
                                    targetDate, null, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                    contentEncoding, null, nocache);
                            FileUtils.copy(result, out);
                        }
                    }
                } else { // no html

View Full Code Here

        if (date == null) return "";
        return dayFormatter.format(date);
    }
    
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final servletProperties prop = new servletProperties();
        final Switchboard sb = (Switchboard)env;
        
        int showLimit = 100;
        if (post != null) {
            showLimit = post.getInt("limit", 100);
            
            if (post.containsKey("clearcrawlqueue")) {
                final int c = sb.crawlQueues.noticeURL.stackSize(NoticedURL.StackType.REMOTE);
                sb.crawlQueues.noticeURL.clear(NoticedURL.StackType.REMOTE);
                try { sb.cleanProfiles(); } catch (final InterruptedException e) { /* Ignore this */}
                /*
                int c = 0;
                while (switchboard.urlPool.noticeURL.stackSize(plasmaCrawlNURL.StackType.LIMIT) > 0) {
                    urlHash = switchboard.urlPool.noticeURL.pop(plasmaCrawlNURL.StackType.LIMIT).hash();
                    if (urlHash != null) { switchboard.urlPool.noticeURL.remove(urlHash); c++; }
                }
                */
                prop.put("info", "3"); // crawling queue cleared
                prop.putNum("info_numEntries", c);
            } else if (post.containsKey("deleteEntry")) {
                final String urlHash = post.get("deleteEntry");
                sb.crawlQueues.noticeURL.removeByURLHash(urlHash.getBytes());
                prop.put("LOCATION","");
                return prop;
            }
        }


        int stackSize = sb.crawlQueues.noticeURL.stackSize(NoticedURL.StackType.REMOTE);
        if (stackSize == 0) {
            prop.put("crawler-queue", "0");
        } else {
            prop.put("crawler-queue", "1");
            final List<Request> crawlerList = sb.crawlQueues.noticeURL.top(NoticedURL.StackType.REMOTE, showLimit);
            
            Request urle;
            boolean dark = true;
            yacySeed initiator;
            String profileHandle;
            CrawlProfile profileEntry;
            int i, showNum = 0;
            for (i = 0; (i < crawlerList.size()) && (showNum < showLimit); i++) {
                urle = crawlerList.get(i);
                if (urle != null && urle.url() != null) {
                    initiator = sb.peers.getConnected((urle.initiator() == null) ? "" : ASCII.String(urle.initiator()));
                    profileHandle = urle.profileHandle();
                    profileEntry = profileHandle == null ? null : sb.crawler.getActive(profileHandle.getBytes());
                    prop.put("crawler-queue_list_" + showNum + "_dark", dark ? "1" : "0");
                    prop.putHTML("crawler-queue_list_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
                    prop.put("crawler-queue_list_" + showNum + "_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
                    prop.put("crawler-queue_list_" + showNum + "_depth", urle.depth());
                    prop.put("crawler-queue_list_" + showNum + "_modified", daydate(urle.appdate()) );
                    prop.putHTML("crawler-queue_list_" + showNum + "_anchor", urle.name());
                    prop.putHTML("crawler-queue_list_" + showNum + "_url", urle.url().toString());
                    prop.put("crawler-queue_list_" + showNum + "_hash", urle.url().hash());
                    dark = !dark;
                    showNum++;
                } else {
                    stackSize--;
                }
            }
            prop.putNum("crawler-queue_show-num", showNum); //showin sjow-num most recent
            prop.putNum("crawler-queue_num", stackSize);//num Entries
            prop.putNum("crawler-queue_list", showNum);
        }
        return prop;
    }

View Full Code Here


    public static serverObjects respond(
            final RequestHeader header,
            final serverObjects post,
            final serverSwitch env) {
        final servletProperties prop = new servletProperties();
        final Switchboard sb = (Switchboard)env;


        // read post for handle
        final String handle = (post == null) ? "" : post.get("handle", "");
        if (post != null) {
            if (post.containsKey("terminate")) try {
                // termination of a crawl: shift the crawl from active to passive
                final CrawlProfile p = sb.crawler.getActive(handle.getBytes());
                if (p != null) sb.crawler.putPassive(handle.getBytes(), p);
                // delete all entries from the crawl queue that are deleted here
                sb.crawler.removeActive(handle.getBytes());
                sb.crawlQueues.noticeURL.removeByProfileHandle(handle, 10000);
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
            if (post.containsKey("delete")) {
                // deletion of a terminated crawl profile
                sb.crawler.removePassive(handle.getBytes());
            }
            if (post.containsKey("deleteTerminatedProfiles")) {
                for (final byte[] h: sb.crawler.getPassive()) {
                    sb.crawler.removePassive(h);
                }
            }
        }


        // generate handle list: first sort by handle name
        CrawlProfile selentry;
        final Map<String, String> orderdHandles = new TreeMap<String, String>();
        for (final byte[] h : sb.crawler.getActive()) {
            selentry = sb.crawler.getActive(h);
            if (selentry != null && !ignoreNames.contains(selentry.name())) {
                orderdHandles.put(selentry.name(), selentry.handle());
            }
        }


        // then write into pop-up menu list
        int count = 0;
        for (final Map.Entry<String, String> NameHandle: orderdHandles.entrySet()) {
            prop.put("profiles_" + count + "_name", NameHandle.getKey());
            prop.put("profiles_" + count + "_handle", NameHandle.getValue());
            if (handle.equals(NameHandle.getValue())) {
                prop.put("profiles_" + count + "_selected", "1");
            }
            count++;
        }
        prop.put("profiles", count);
        selentry = sb.crawler.getActive(handle.getBytes());
        assert selentry == null || selentry.handle() != null;
        // read post for change submit
        if ((post != null) && (selentry != null)) {
            if (post.containsKey("submit")) {
                try {
                  Pattern.compile(post.get(CrawlProfile.FILTER_URL_MUSTMATCH, CrawlProfile.MATCH_ALL_STRING));
                  Pattern.compile(post.get(CrawlProfile.FILTER_URL_MUSTNOTMATCH, CrawlProfile.MATCH_NEVER_STRING));
                    final Iterator<eentry> lit = labels.iterator();
                    eentry tee;
                    while (lit.hasNext()) {
                        tee = lit.next();
                        final String cval = selentry.get(tee.name);
                        final String val = (tee.type == eentry.BOOLEAN) ? Boolean.toString(post.containsKey(tee.name)) : post.get(tee.name, cval);
                        if (!cval.equals(val)) {
                            selentry.put(tee.name, val);
                            sb.crawler.putActive(selentry.handle().getBytes(), selentry);
                        }
                    }
                } catch (final Exception ex) {
                    Log.logException(ex);
                    prop.put("error", "1");
                    prop.putHTML("error_message", ex.getMessage());
                }
            }
        }


        // generate crawl profile table
        count = 0;
        boolean dark = true;
        final int domlistlength = (post == null) ? 160 : post.getInt("domlistlength", 160);
        CrawlProfile profile;
        // put active crawls into list
        for (final byte[] h: sb.crawler.getActive()) {
            profile = sb.crawler.getActive(h);
            putProfileEntry(prop, sb.crawlStacker, profile, true, dark, count, domlistlength);
            dark = !dark;
            count++;
        }
        // put passive crawls into list
        boolean existPassiveCrawls = false;
        for (final byte[] h: sb.crawler.getPassive()) {
            profile = sb.crawler.getPassive(h);
            putProfileEntry(prop, sb.crawlStacker, profile, false, dark, count, domlistlength);
            dark = !dark;
            count++;
            existPassiveCrawls = true;
        }
        prop.put("crawlProfiles", count);


        prop.put("existPassiveCrawls", existPassiveCrawls ? "1" : "0");


        // generate edit field
        if (selentry == null) {
          prop.put("edit", "0");
        } else {
            prop.put("edit", "1");
            prop.put("edit_name", selentry.name());
            prop.put("edit_handle", selentry.handle());
            final Iterator<eentry> lit = labels.iterator();
            count = 0;
            while (lit.hasNext()) {
                final eentry ee = lit.next();
                final String val = selentry.get(ee.name);
                prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly", ee.readonly ? "1" : "0");
                prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_name", ee.name);
                prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_label", ee.label);
                prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_type", ee.type);
                if (ee.type == eentry.BOOLEAN) {
                    prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_type_checked",
                            Boolean.parseBoolean(val) ? "1" : "0");
                } else {
                    prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_type_value", val);
                }
                count++;
            }
            prop.put("edit_entries", count);
        }


        return prop;
    }

View Full Code Here


    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final boolean authenticated = sb.adminAuthenticated(header) >= 2;
        final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
        final servletProperties prop = new servletProperties();


        prop.put("display", display);


        String default_left = sb.getConfig("compare_yacy.left", defaultsearchL);
        if (!searchengines.containsKey(default_left)) default_left = defaultsearchL;
        String default_right = sb.getConfig("compare_yacy.right", defaultsearchR);
        if (!searchengines.containsKey(default_right)) default_right = defaultsearchR;


        if (post != null) {
            if (searchengines.get(post.get("left", default_left)) != null) {
                default_left = post.get("left", default_left);
                sb.setConfig("compare_yacy.left", default_left);
            }
            if (searchengines.get(post.get("right", default_right)) != null) {
                default_right = post.get("right", default_right);
                sb.setConfig("compare_yacy.right", default_right);
            }
        }


        prop.put("searchengines", order.length);
        String name;
        for (int i = 0; i < order.length; i++) {
            name = order[i];
            prop.putHTML("searchengines_" + i + "_searchengine", name);
            prop.put("searchengines_" + i + "_leftengine", name.equals(default_left) ? 1 : 0);
            prop.put("searchengines_" + i + "_rightengine", name.equals(default_right) ? 1 : 0);
        }


        prop.putHTML("search_left", searchengines.get(default_left));
        prop.putHTML("search_right", searchengines.get(default_right));


        if (post == null || post.get("query", "").length() == 0) {
            prop.put("search", 0);
            prop.put("search_query", "");
            return prop;
        }


        prop.put("search", 1);
        prop.putHTML("search_query", post.get("query", ""));


        // return rewrite properties
        return prop;
    }

View Full Code Here


    private static final int meanMax = 30;


    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final servletProperties prop = new servletProperties();


        final String ext = header.get("EXT", "");
        final boolean json = ext.equals("json");
        final boolean xml = ext.equals("xml");
        final boolean more = post != null && post.containsKey("more");


        // get query
        final String originalquerystring = (post == null) ? "" : post.get("query", post.get("q", "")).trim();
        final String querystring =  originalquerystring.replace('+', ' ');
        final int timeout = (post == null) ? 300 : post.getInt("timeout", 300);
        final int count = (post == null) ? 20 : post.getInt("count", 20);


        // get segment
        final Segment indexSegment;
        if (post != null && post.containsKey("segment")) {
            final String segmentName = post.get("segment");
            if (sb.indexSegments.segmentExist(segmentName)) {
                indexSegment = sb.indexSegments.segment(segmentName);
            } else {
                // take default segment
                indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
            }
        } else {
            // take default segment
            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }


        int c = 0;
        if (more ||
                (indexSegment != null &&
                !indexSegment.termIndex().has(Word.word2hash(querystring))))
        {
            final DidYouMean didYouMean = new DidYouMean(indexSegment.termIndex(), new StringBuilder(querystring));
            final Iterator<StringBuilder> meanIt = didYouMean.getSuggestions(timeout, count).iterator();
            String suggestion;
            //[#[query]#,[#{suggestions}##[text]##(eol)#,::#(/eol)##{/suggestions}#]]
            while (c < meanMax && meanIt.hasNext()) {
                suggestion = meanIt.next().toString();
                if (json) {
                    prop.putJSON("suggestions_" + c + "_text", suggestion);
                } else if (xml) {
                    prop.putXML("suggestions_" + c + "_text", suggestion);
                } else {
                    prop.putHTML("suggestions_" + c + "_text", suggestion);
                }
                prop.put("suggestions_" + c + "_eol", 0);
                c++;
            }
        }


        if (c > 0) {
            prop.put("suggestions_" + (c - 1) + "_eol", 1);
        }
        prop.put("suggestions", c);
        if (json) {
            prop.putJSON("query", originalquerystring);
        } else if (xml) {
            prop.putXML("query", originalquerystring);
        } else {
            prop.putHTML("query", originalquerystring);
        }


        // Adding CORS Access header for xml output
        if (xml) {
            final ResponseHeader outgoingHeader = new ResponseHeader();
            outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");
            prop.setOutgoingHeader(outgoingHeader);
        }


        // return rewrite properties
        return prop;
    }

View Full Code Here


        // get query
        final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
        String querystring =  originalquerystring.replace('+', ' ').replace('*', ' ').trim();
        CacheStrategy snippetFetchStrategy = (post == null) ? null : CacheStrategy.parse(post.get("verify", "cacheonly"));
        final servletProperties prop = new servletProperties();
        prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);


        // get segment
        Segment indexSegment = null;
        if (post != null && post.containsKey("segment")) {
            final String segmentName = post.get("segment");
            if (sb.indexSegments.segmentExist(segmentName)) {
                indexSegment = sb.indexSegments.segment(segmentName);
            }
        } else {
            // take default segment
            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }


        final String EXT = header.get("EXT", "");
        final boolean rss = EXT.equals("rss");
        final boolean json = EXT.equals("json");
        prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
        prop.put("promoteSearchPageGreeting.homepage", sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, ""));
        prop.put("promoteSearchPageGreeting.smallImage", sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
        if (post == null || indexSegment == null || env == null || !searchAllowed) {
            // we create empty entries for template strings
            prop.put("searchagain", "0");
            prop.put("former", "");
            prop.put("count", "10");
            prop.put("offset", "0");
            prop.put("resource", "global");
            prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*"));
            prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", ""));
            prop.put("tenant", (post == null) ? "" : post.get("tenant", ""));
            prop.put("indexof", "off");
            prop.put("constraint", "");
            prop.put("cat", "href");
            prop.put("depth", "0");
            prop.put("search.verify", (post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));
            prop.put("search.navigation", (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
            prop.put("contentdom", "text");
            prop.put("contentdomCheckText", "1");
            prop.put("contentdomCheckAudio", "0");
            prop.put("contentdomCheckVideo", "0");
            prop.put("contentdomCheckImage", "0");
            prop.put("contentdomCheckApp", "0");
            prop.put("excluded", "0");
            prop.put("results", "");
            prop.put("resultTable", "0");
            prop.put("num-results", searchAllowed ? "0" : "4");
            prop.put("num-results_totalcount", 0);
            prop.put("num-results_offset", 0);
            prop.put("num-results_itemsPerPage", 10);
            prop.put("geoinfo", "0");
            prop.put("rss_queryenc", "");
            prop.put("meanCount", 5);
            return prop;
        }


        // check for JSONP
        if (post.containsKey("callback")) {
          final String jsonp = post.get("callback")+ "([";
          prop.put("jsonp-start", jsonp);
          prop.put("jsonp-end", "])");
        } else {
          prop.put("jsonp-start", "");
          prop.put("jsonp-end", "");
        }


        // Adding CORS Access header for yacysearch.rss output
        if (rss) {
            final ResponseHeader outgoingHeader = new ResponseHeader();
            outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");
            prop.setOutgoingHeader(outgoingHeader);
        }


        // collect search attributes


        int maximumRecords = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 5000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
        int startRecord = post.getInt("startRecord", post.getInt("offset", 0));


        boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0;
        final boolean indexof = (post != null && post.get("indexof","").equals("on"));


        final String originalUrlMask;
        if (post.containsKey("urlmaskfilter")) {
            originalUrlMask = post.get("urlmaskfilter", ".*");
        } else {
            originalUrlMask = ".*";
        }


        String prefermask = (post == null) ? "" : post.get("prefermaskfilter", "");
        if (!prefermask.isEmpty() && prefermask.indexOf(".*",0) < 0) {
            prefermask = ".*" + prefermask + ".*";
        }


        Bitfield constraint = (post != null && post.containsKey("constraint") && !post.get("constraint", "").isEmpty()) ? new Bitfield(4, post.get("constraint", "______")) : null;
        if (indexof) {
            constraint = new Bitfield(4);
            constraint.set(Condenser.flag_cat_indexof, true);
        }


        // SEARCH
        final boolean clustersearch = sb.isRobinsonMode() && (sb.getConfig("cluster.mode", "").equals("privatecluster") || sb.getConfig("cluster.mode", "").equals("publiccluster"));
        final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) || sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true) || clustersearch;
        global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global searchnn


        // increase search statistic counter
        if (!global) {
            // we count only searches on the local peer here, because global searches
            // are counted on the target peer to preserve privacy of the searcher
            if (authenticated) {
                // local or authenticated search requests are counted separately
                // because they are not part of a public available peer statistic
                sb.searchQueriesRobinsonFromLocal++;
            } else {
                // robinson-searches from non-authenticated requests are public
                // and may be part of the public available statistic
                sb.searchQueriesRobinsonFromRemote++;
            }
        }


        // find search domain
        final ContentDomain contentdom = ContentDomain.contentdomParser(post == null ? "text" : post.get("contentdom", "text"));


        // patch until better search profiles are available
        if (contentdom == ContentDomain.TEXT) {
            if (maximumRecords > 50) maximumRecords = 10;
        } else {
            if (maximumRecords <= 32) maximumRecords = 64;
        }


        // check the search tracker
        TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
        if (trackerHandles == null) {
            trackerHandles = new TreeSet<Long>();
        }
        boolean block = false;
        if (Domains.matchesList(client, sb.networkBlacklist)) {
            global = false;
            if (snippetFetchStrategy != null) {
                snippetFetchStrategy = null;
            }
            block = true;
            Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search");
        } else if (Domains.matchesList(client, sb.networkWhitelist)) {
            Log.logInfo("LOCAL_SEARCH", "ACCESS CONTROL: WHITELISTED CLIENT FROM " + client + " gets no search restrictions");
        } else if (!authenticated && !localhostAccess) {
            // in case that we do a global search or we want to fetch snippets, we check for DoS cases
            synchronized (trackerHandles) {
                final int accInThreeSeconds = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size();
                final int accInOneMinute = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size();
                final int accInTenMinutes = trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size();
                // protections against too strong YaCy network load, reduces remote search
                if (global) {
                    if (accInTenMinutes >= 60 || accInOneMinute >= 6 || accInThreeSeconds >= 1) {
                        global = false;
                        Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed global search");
                    }
                }
                // protection against too many remote server snippet loads (protects traffic on server)
                if (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline()) {
                    if (accInTenMinutes >= 20 || accInOneMinute >= 4 || accInThreeSeconds >= 1) {
                        snippetFetchStrategy = CacheStrategy.CACHEONLY;
                        Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed remote snippet loading");
                    }
                }
                // general load protection
                if (accInTenMinutes >= 3000 || accInOneMinute >= 600 || accInThreeSeconds >= 60) {
                    block = true;
                    Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM " + client + ": " + accInThreeSeconds + "/3s, " + accInOneMinute + "/60s, " + accInTenMinutes + "/600s, " + " requests, disallowed search");
                }
            }
        }


        if (!block && (post == null || post.get("cat", "href").equals("href"))) {
            String urlmask = null;


            // check available memory and clean up if necessary
            if (!MemoryControl.request(8000000L, false)) {
                indexSegment.urlMetadata().clearCache();
                SearchEventCache.cleanupEvents(true);
            }


            final RankingProfile ranking = sb.getRanking();
            final StringBuilder modifier = new StringBuilder(20);


            if (querystring.indexOf("/near",0) >= 0) {
              querystring = querystring.replace("/near", "");
              ranking.coeff_worddistance = RankingProfile.COEFF_MAX;
              modifier.append("/near ");
            }
            if (querystring.indexOf("/date",0) >= 0) {
                querystring = querystring.replace("/date", "");
                ranking.coeff_date = RankingProfile.COEFF_MAX;
                modifier.append("/date ");
            }
            if (querystring.indexOf("/http",0) >= 0) {
                querystring = querystring.replace("/http", "");
                urlmask = "https?://.*";
                modifier.append("/http ");
            }
            if (querystring.indexOf("/https",0) >= 0) {
                querystring = querystring.replace("/https", "");
                urlmask = "https?://.*";
                modifier.append("/https ");
            }
            if (querystring.indexOf("/ftp",0) >= 0) {
                querystring = querystring.replace("/ftp", "");
                urlmask = "ftp://.*";
                modifier.append("/ftp ");
            }
            if (querystring.indexOf("/smb",0) >= 0) {
                querystring = querystring.replace("/smb", "");
                urlmask = "smb://.*";
                modifier.append("/smb ");
            }
            if (querystring.indexOf("/file",0) >= 0) {
                querystring = querystring.replace("/file", "");
                urlmask = "file://.*";
                modifier.append("/file ");
            }
            if (querystring.indexOf("/location",0) >= 0) {
                querystring = querystring.replace("/location", "");
                if (constraint == null) {
                    constraint = new Bitfield(4);
                }
                constraint.set(Condenser.flag_cat_haslocation, true);
                modifier.append("/location ");
            }
            final int lrp = querystring.indexOf("/language/",0);
            String language = "";
            if (lrp >= 0) {
                if (querystring.length() >= (lrp + 12)) {
                    language = querystring.substring(lrp + 10, lrp + 12);
                }
                querystring = querystring.replace("/language/" + language, "");
                language = language.toLowerCase();
                modifier.append("/language/").append(language).append(" ");
            }
            final int inurl = querystring.indexOf("inurl:",0);
            if (inurl >= 0) {
                int ftb = querystring.indexOf(' ', inurl);
                if (ftb == -1) {
                    ftb = querystring.length();
                }
                final String urlstr = querystring.substring(inurl + 6, ftb);
                querystring = querystring.replace("inurl:" + urlstr, "");
                if (!urlstr.isEmpty()) {
                    urlmask = urlmask == null ? ".*" + urlstr + ".*" : urlmask + urlstr + ".*";
                }
                modifier.append("inurl:").append(urlstr).append(" ");
            }
            final int filetype = querystring.indexOf("filetype:",0);
            if (filetype >= 0) {
                int ftb = querystring.indexOf(' ', filetype);
                if (ftb == -1) {
                    ftb = querystring.length();
                }
                String ft = querystring.substring(filetype + 9, ftb);
                querystring = querystring.replace("filetype:" + ft, "");
                while (!ft.isEmpty() && ft.charAt(0) == '.') ft = ft.substring(1);
                if (!ft.isEmpty()) {
                    if (urlmask == null) {
                        urlmask = ".*\\." + ft;
                    } else {
                        urlmask = urlmask + ".*\\." + ft;
                    }
                }
                modifier.append("filetype:").append(ft).append(" ");
            }
            String tenant = null;
            if (post.containsKey("tenant")) {
                tenant = post.get("tenant");
                if (tenant != null && tenant.isEmpty()) {
                    tenant = null;
                }
                if (tenant != null) {
                    if (urlmask == null) {
                        urlmask = ".*" + tenant + ".*";
                    } else urlmask = ".*" + tenant + urlmask;
                }
            }
            final int site = querystring.indexOf("site:",0);
            String sitehash = null;
            String sitehost = null;
            if (site >= 0) {
                int ftb = querystring.indexOf(' ', site);
                if (ftb == -1) {
                    ftb = querystring.length();
                }
                sitehost = querystring.substring(site + 5, ftb);
                querystring = querystring.replace("site:" + sitehost, "");
                while (sitehost.length() > 0 && sitehost.charAt(0) == '.') {
                    sitehost = sitehost.substring(1);
                }
                while (sitehost.endsWith(".")) {
                    sitehost = sitehost.substring(0, sitehost.length() - 1);
                }
                sitehash = DigestURI.hosthash(sitehost);
                modifier.append("site:").append(sitehost).append(" ");
            }


            final int heuristicScroogle = querystring.indexOf("/heuristic/scroogle",0);
            if (heuristicScroogle >= 0) {
                querystring = querystring.replace("/heuristic/scroogle", "");
                modifier.append("/heuristic/scroogle ");
            }


            final int heuristicBlekko = querystring.indexOf("/heuristic/blekko",0);
            if (heuristicBlekko >= 0) {
                querystring = querystring.replace("/heuristic/blekko", "");
                modifier.append("/heuristic/blekko ");
            }


            final int authori = querystring.indexOf("author:",0);
          String authorhash = null;
            if (authori >= 0) {
              // check if the author was given with single quotes or without
              final boolean quotes = (querystring.charAt(authori + 7) == (char) 39);
              String author;
              if (quotes) {
                    int ftb = querystring.indexOf((char) 39, authori + 8);
                    if (ftb == -1) {
                        ftb = querystring.length() + 1;
                    }
                    author = querystring.substring(authori + 8, ftb);
                    querystring = querystring.replace("author:'" + author + "'", "");
                    modifier.append("author:'").append(author).append("' ");
              } else {
                    int ftb = querystring.indexOf(' ', authori);
                    if (ftb == -1) {
                        ftb = querystring.length();
                    }
                    author = querystring.substring(authori + 7, ftb);
                    querystring = querystring.replace("author:" + author, "");
                    modifier.append("author:").append(author).append(" ");
              }
              authorhash = ASCII.String(Word.word2hash(author));
            }
            final int tld = querystring.indexOf("tld:",0);
            if (tld >= 0) {
                int ftb = querystring.indexOf(' ', tld);
                if (ftb == -1) {
                    ftb = querystring.length();
                }
                String domain = querystring.substring(tld + 4, ftb);
                querystring = querystring.replace("tld:" + domain, "");
                modifier.append("tld:").append(domain).append(" ");
                while (domain.length() > 0 && domain.charAt(0) == '.') {
                    domain = domain.substring(1);
                }
                if (domain.indexOf('.',0) < 0) {
                    domain = "\\." + domain;
                } // is tld
                if (domain.length() > 0) {
                    urlmask = "[a-zA-Z]*://[^/]*" + domain + "/.*" + ((urlmask != null) ? urlmask : "");
                }
            }
            if (urlmask == null || urlmask.isEmpty()) {
                urlmask = originalUrlMask;
            } //if no urlmask was given


            // read the language from the language-restrict option 'lr'
            // if no one is given, use the user agent or the system language as default
            language = (post == null) ? language : post.get("lr", language);
            if (language.startsWith("lang_")) {
                language = language.substring(5);
            }
            if (!ISO639.exists(language)) {
                // find out language of the user by reading of the user-agent string
                String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE);
                if (agent == null) {
                    agent = System.getProperty("user.language");
                }
                language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent);
                if (language == null) {
                    language = "en";
                }
            }


            // navigation
            final String navigation = (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "");


            // the query
            final TreeSet<String>[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute


            final int maxDistance = (querystring.indexOf('"',0) >= 0) ? query.length - 1 : Integer.MAX_VALUE;


            // filter out stopwords
            final SortedSet<String> filtered = SetTools.joinConstructive(query[0], Switchboard.stopwords);
            if (!filtered.isEmpty()) {
                SetTools.excludeDestructive(query[0], Switchboard.stopwords);
            }


            // if a minus-button was hit, remove a special reference first
            if (post != null && post.containsKey("deleteref")) {
                try {
                    if (!sb.verifyAuthentication(header)) {
                        prop.put("AUTHENTICATE", "admin log-in"); // force log-in
                        return prop;
                    }


                    // delete the index entry locally
                    final String delHash = post.get("deleteref", ""); // urlhash
                    indexSegment.termIndex().remove(Word.words2hashesHandles(query[0]), delHash.getBytes());


                    // make new news message with negative voting
                    if (!sb.isRobinsonMode()) {
                        final Map<String, String> map = new HashMap<String, String>();
                        map.put("urlhash", delHash);
                        map.put("vote", "negative");
                        map.put("refid", "");
                        sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
                    }


                    // delete the search history since this still shows the entry
                    SearchEventCache.delete(delHash);
                } catch (final IOException e) {
                    Log.logException(e);
                }
            }


            // if a plus-button was hit, create new voting message
            if (post != null && post.containsKey("recommendref")) {
                if (!sb.verifyAuthentication(header)) {
                    prop.put("AUTHENTICATE", "admin log-in"); // force log-in
                    return prop;
                }
                final String recommendHash = post.get("recommendref", ""); // urlhash
                final URIMetadataRow urlentry = indexSegment.urlMetadata().load(UTF8.getBytes(recommendHash));
                if (urlentry != null) {
                    final URIMetadataRow.Components metadata = urlentry.metadata();
                    Document[] documents = null;
                    try {
                        documents = sb.loader.loadDocuments(sb.loader.request(metadata.url(), true, false), CacheStrategy.IFEXIST, 5000, Integer.MAX_VALUE);
                    } catch (final IOException e) {
                    } catch (final Parser.Failure e) {
                    }
                    if (documents != null) {
                        // create a news message
                        final Map<String, String> map = new HashMap<String, String>();
                        map.put("url", metadata.url().toNormalform(false, true).replace(',', '|'));
                        map.put("title", metadata.dc_title().replace(',', ' '));
                        map.put("description", documents[0].dc_title().replace(',', ' '));
                        map.put("author", documents[0].dc_creator());
                        map.put("tags", documents[0].dc_subject(' '));
                        sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_SURFTIPP_ADD, map);
                        documents[0].close();
                    }
                }
            }


            // if a bookmarks-button was hit, create new bookmark entry
            if (post != null && post.containsKey("bookmarkref")) {
                if (!sb.verifyAuthentication(header)) {
                    prop.put("AUTHENTICATE", "admin log-in"); // force log-in
                    return prop;
                }
                final String bookmarkHash = post.get("bookmarkref", ""); // urlhash
                final URIMetadataRow urlentry = indexSegment.urlMetadata().load(UTF8.getBytes(bookmarkHash));
                if (urlentry != null) {
                    final URIMetadataRow.Components metadata = urlentry.metadata();
                    try {
                        sb.tables.bookmarks.createBookmark(sb.loader, metadata.url(), YMarkTables.USER_ADMIN, true, "searchresult", "/search");
                    } catch (final Throwable e) {
                    }
                }
            }


            // do the search
            final HandleSet queryHashes = Word.words2hashesHandles(query[0]);
            final Pattern snippetPattern = QueryParams.stringSearchPattern(originalquerystring);


            // check filters
            try {
                Pattern.compile(urlmask);
            } catch (final PatternSyntaxException ex) {
                Log.logWarning("SEARCH", "Illegal URL mask, not a valid regex: " + urlmask);
                prop.put("urlmaskerror", 1);
                prop.putHTML("urlmaskerror_urlmask", urlmask);
                urlmask = ".*";
            }


            try {
                Pattern.compile(prefermask);
            } catch (final PatternSyntaxException ex) {
                Log.logWarning("SEARCH", "Illegal prefer mask, not a valid regex: " + prefermask);
                prop.put("prefermaskerror", 1);
                prop.putHTML("prefermaskerror_prefermask", prefermask);
                prefermask = "";
            }


            final QueryParams theQuery = new QueryParams(
                    originalquerystring,
                    queryHashes,
                    Word.words2hashesHandles(query[1]),
                    Word.words2hashesHandles(query[2]),
                    snippetPattern,
                    tenant,
                    modifier.toString().trim(),
                    maxDistance,
                    prefermask,
                    contentdom,
                    language,
                    navigation,
                    snippetFetchStrategy,
                    maximumRecords,
                    startRecord,
                    urlmask,
                    clustersearch && global ? QueryParams.Searchdom.CLUSTER :
                    (global && indexReceiveGranted ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL),
                    20,
                    constraint,
                    true,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    authenticated,
                    indexSegment,
                    ranking,
                    header.get(RequestHeader.USER_AGENT, ""),
                    sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
            EventTracker.delete(EventTracker.EClass.SEARCH);
            EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);


            // tell all threads to do nothing for a specific time
            sb.intermissionAllThreads(3000);


            // filter out words that appear in bluelist
            theQuery.filterOut(Switchboard.blueList);


            // log
            Log.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + QueryParams.hashSet2hashString(theQuery.queryHashes) + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
            EventChannel.channels(EventChannel.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
            final long timestamp = System.currentTimeMillis();


            // create a new search event
            if (SearchEventCache.getEvent(theQuery.id(false)) == null) {
                theQuery.setOffset(0); // in case that this is a new search, always start without a offset
                startRecord = 0;
            }
            final SearchEvent theSearch = SearchEventCache.getEvent(
                theQuery, sb.peers, sb.tables, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false, sb.loader,
                (int) sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),
                      sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)),
                (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0),
                (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));


            if (startRecord == 0) {
                if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) {
                    sb.heuristicSite(theSearch, sitehost);
                }
                if ((heuristicScroogle >= 0  || sb.getConfigBool("heuristic.scroogle", false)) && authenticated) {
                    sb.heuristicScroogle(theSearch);
                }
                if ((heuristicBlekko >= 0  || sb.getConfigBool("heuristic.blekko", false)) && authenticated) {
                    sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko");
                }
            }


            // log
            Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
                    "local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
                    "local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
                    "local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
                    "remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " +
                    (System.currentTimeMillis() - timestamp) + " ms");


            // prepare search statistics
            theQuery.resultcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
            theQuery.searchtime = System.currentTimeMillis() - timestamp;
            theQuery.urlretrievaltime = theSearch.result().getURLRetrievalTime();
            theQuery.snippetcomputationtime = theSearch.result().getSnippetComputationTime();
            AccessTracker.add(AccessTracker.Location.local, theQuery);


            // check suggestions
            final int meanMax = (post != null) ? post.getInt("meanCount", 0) : 0;


            prop.put("meanCount", meanMax);
            if (meanMax > 0 && !json && !rss) {
                final DidYouMean didYouMean = new DidYouMean(indexSegment.termIndex(), new StringBuilder(querystring));
              final Iterator<StringBuilder> meanIt = didYouMean.getSuggestions(100, 5).iterator();
                int meanCount = 0;
                String suggestion;
                while( meanCount<meanMax && meanIt.hasNext()) {
                    suggestion = meanIt.next().toString();
                    prop.put("didYouMean_suggestions_"+meanCount+"_word", suggestion);
                    prop.put("didYouMean_suggestions_"+meanCount+"_url",
                            QueryParams.navurl("html", 0, theQuery, suggestion, originalUrlMask.toString(), theQuery.navigators).toString()
                   );
                    prop.put("didYouMean_suggestions_"+meanCount+"_sep","|");
                    meanCount++;
                }
                prop.put("didYouMean_suggestions_"+(meanCount-1)+"_sep","");
                prop.put("didYouMean", meanCount>0 ? 1:0);
                prop.put("didYouMean_suggestions", meanCount);
            } else {
                prop.put("didYouMean", 0);
            }


            // find geographic info
            final SortedSet<Location> coordinates = LibraryProvider.geoLoc.find(originalquerystring, false);
            if (coordinates == null || coordinates.isEmpty() || startRecord > 0) {
                prop.put("geoinfo", "0");
            } else {
                int i = 0;
                for (final Location c: coordinates) {
                    prop.put("geoinfo_loc_" + i + "_lon", Math.round(c.lon() * 10000.0f) / 10000.0f);
                    prop.put("geoinfo_loc_" + i + "_lat", Math.round(c.lat() * 10000.0f) / 10000.0f);
                    prop.put("geoinfo_loc_" + i + "_name", c.getName());
                    i++;
                    if (i >= 10) break;
                }
                prop.put("geoinfo_loc", i);
                prop.put("geoinfo", "1");
            }


            // update the search tracker
            try {
                synchronized (trackerHandles) {
                    trackerHandles.add(theQuery.time);
                    while (trackerHandles.size() > 600) {
                        if (!trackerHandles.remove(trackerHandles.first())) break;
                    }
                }
                sb.localSearchTracker.put(client, trackerHandles);
              if (sb.localSearchTracker.size() > 100) {
                    sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement());
                }
              if (MemoryControl.shortStatus()) sb.localSearchTracker.clear();
            } catch (final Exception e) {
                Log.logException(e);
            }


            final int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
            prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1);
            prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.getQuery().itemsPerPage > indexcount ? startRecord + indexcount % theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage, true));
            prop.put("num-results_itemsPerPage", maximumRecords);
            prop.put("num-results_totalcount", Formatter.number(indexcount, true));
            prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0");
            prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true));
            prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.getRankingResult().getMissCount(), true));
            prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.getRankingResult().getRemoteResourceSize(), true));
            prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
            prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));


            // compose page navigation
            final StringBuilder resnav = new StringBuilder(200);
            final int thispage = startRecord / theQuery.displayResults();
            if (thispage == 0) {
              resnav.append("<img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" />&nbsp;");
            } else {
              resnav.append("<a id=\"prevpage\" href=\"");
                resnav.append(QueryParams.navurl("html", thispage - 1, theQuery, null, originalUrlMask, navigation).toString());
              resnav.append("\"><img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" /></a>&nbsp;");
            }
            final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.displayResults()));


            for (int i = 0; i < numberofpages; i++) {
                if (i == thispage) {
                    resnav.append("<img src=\"env/grafics/navs");
                    resnav.append(i + 1);
                    resnav.append(".gif\" alt=\"page");
                    resnav.append(i + 1);
                    resnav.append("\" width=\"16\" height=\"16\" />&nbsp;");
                } else {
                    resnav.append("<a href=\"");
                    resnav.append(QueryParams.navurl("html", i, theQuery, null, originalUrlMask, navigation).toString());
                    resnav.append("\"><img src=\"env/grafics/navd");
                    resnav.append(i + 1);
                    resnav.append(".gif\" alt=\"page");
                    resnav.append(i + 1);
                    resnav.append("\" width=\"16\" height=\"16\" /></a>&nbsp;");
                }
            }
            if (thispage >= numberofpages) {
              resnav.append("<img src=\"env/grafics/navdr.gif\" alt=\"arrowright\" width=\"16\" height=\"16\" />");
            } else {
                resnav.append("<a id=\"nextpage\" href=\"");
                resnav.append(QueryParams.navurl("html", thispage + 1, theQuery, null, originalUrlMask, navigation).toString());
                resnav.append("\"><img src=\"env/grafics/navdr.gif\" alt=\"arrowright\" width=\"16\" height=\"16\" /></a>");
            }
            final String resnavs = resnav.toString();
            prop.put("num-results_resnav", resnavs);
            prop.put("pageNavBottom", (indexcount - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
            prop.put("pageNavBottom_resnav", resnavs);


            // generate the search result lines; the content will be produced by another servlet
            for (int i = 0; i < theQuery.displayResults(); i++) {
                prop.put("results_" + i + "_item", startRecord + i);
                prop.put("results_" + i + "_eventID", theQuery.id(false));
            }
            prop.put("results", theQuery.displayResults());
            prop.put("resultTable", (contentdom == ContentDomain.APP || contentdom == ContentDomain.AUDIO || contentdom == ContentDomain.VIDEO) ? 1 : 0);
            prop.put("eventID", theQuery.id(false)); // for bottomline


            // process result of search
            if (!filtered.isEmpty()) {
                prop.put("excluded", "1");
                prop.putHTML("excluded_stopwords", filtered.toString());
            } else {
                prop.put("excluded", "0");
            }


            if (prop == null || prop.isEmpty()) {
                if (post.get("query", post.get("search", "")).length() < 2) {
                    prop.put("num-results", "2"); // no results - at least 2 chars
                } else {
                    prop.put("num-results", "1"); // no results
                }
            } else {
                prop.put("num-results", "3");
            }


            prop.put("cat", "href");
            prop.put("depth", "0");


            // adding some additional properties needed for the rss feed
            String hostName = header.get("Host", "localhost");
            if (hostName.indexOf(':',0) == -1) {
                hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090"));
            }
            prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html");
            prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif");
        }


        prop.put("searchagain", global ? "1" : "0");
        prop.putHTML("former", originalquerystring);
        prop.put("count", maximumRecords);
        prop.put("offset", startRecord);
        prop.put("resource", global ? "global" : "local");
        prop.putHTML("urlmaskfilter", originalUrlMask);
        prop.putHTML("prefermaskfilter", prefermask);
        prop.put("indexof", (indexof) ? "on" : "off");
        prop.put("constraint", (constraint == null) ? "" : constraint.exportB64());
        prop.put("search.verify", snippetFetchStrategy == null ? sb.getConfig("search.verify", "iffresh") : snippetFetchStrategy.toName());
        prop.put("search.navigation", (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
        prop.put("contentdom", (post == null ? "text" : post.get("contentdom", "text")));
        prop.put("searchdomswitches", sb.getConfigBool("search.text", true) || sb.getConfigBool("search.audio", true) || sb.getConfigBool("search.video", true) || sb.getConfigBool("search.image", true) || sb.getConfigBool("search.app", true) ? 1 : 0);
        prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0);
        prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0);
        prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0);
        prop.put("searchdomswitches_searchimage", sb.getConfigBool("search.image", true) ? 1 : 0);
        prop.put("searchdomswitches_searchapp", sb.getConfigBool("search.app", true) ? 1 : 0);
        prop.put("searchdomswitches_searchtext_check", (contentdom == ContentDomain.TEXT) ? "1" : "0");
        prop.put("searchdomswitches_searchaudio_check", (contentdom == ContentDomain.AUDIO) ? "1" : "0");
        prop.put("searchdomswitches_searchvideo_check", (contentdom == ContentDomain.VIDEO) ? "1" : "0");
        prop.put("searchdomswitches_searchimage_check", (contentdom == ContentDomain.IMAGE) ? "1" : "0");
        prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0");


        // copy properties for "more options" link
        prop.put("searchdomswitches_count", prop.get("count"));
        prop.put("searchdomswitches_urlmaskfilter", prop.get("urlmaskfilter"));
        prop.put("searchdomswitches_prefermaskfilter", prop.get("prefermaskfilter"));
        prop.put("searchdomswitches_cat", prop.get("cat"));
        prop.put("searchdomswitches_constraint", prop.get("constraint"));
        prop.put("searchdomswitches_contentdom", prop.get("contentdom"));
        prop.put("searchdomswitches_former", prop.get("former"));
        prop.put("searchdomswitches_meanCount", prop.get("meanCount"));


        // for RSS: don't HTML encode some elements
        prop.putXML("rss_query", originalquerystring);
        prop.putXML("rss_queryenc", originalquerystring.replace(' ', '+'));


        sb.localSearchLastAccess = System.currentTimeMillis();


        // return rewrite properties
        return prop;

View Full Code Here


//dummy class
public class Help {


    public static servletProperties respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch env) {
        final servletProperties prop = new servletProperties();
        return prop;
    }

View Full Code Here

import de.anomic.server.servletProperties;


public class CacheResource_p {


    public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final servletProperties prop = new servletProperties();
        prop.put("resource", new byte[0]);
        
        if (post == null) return prop;
        
        final String u = post.get("url", "");
        DigestURI url;
        try {
            url = new DigestURI(u);
        } catch (MalformedURLException e) {
            Log.logException(e);
            return prop;
        }
        
        byte[] resource = null;
        resource = Cache.getContent(url.hash());
        if (resource == null) return prop;
        
        // check request type
        if (header.get("EXT", "html").equals("png")) {
            // a png was requested
            return ImageParser.parse(u, resource);
        } else {
            // get response header and set mime type
            ResponseHeader responseHeader = Cache.getResponseHeader(url.hash());
            String resMime = responseHeader == null ? null : responseHeader.mime();
            if (resMime != null) {
                final ResponseHeader outgoingHeader = new ResponseHeader();
                outgoingHeader.put(HeaderFramework.CONTENT_TYPE, resMime);
                prop.setOutgoingHeader(outgoingHeader);
            }        
    
            // add resource
            prop.put("resource", resource);
            return prop;
        }
    }

View Full Code Here


public class ConfigRobotsTxt_p {
    
    public static servletProperties respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final servletProperties prop = new servletProperties();
        
        final RobotsTxtConfig rbc = ((Switchboard)env).robotstxtConfig;
        prop.put("clientname", sb.peers.mySeed().getPublicAddress());
        
        if (post != null) {
            if (post.containsKey("save")) {
                rbc.setAllDisallowed(post.containsKey(RobotsTxtConfig.ALL));
                rbc.setBlogDisallowed(post.containsKey(RobotsTxtConfig.BLOG));
                rbc.setBookmarksDisallowed(post.containsKey(RobotsTxtConfig.BOOKMARKS));
                rbc.setDirsDisallowed(post.containsKey(RobotsTxtConfig.DIRS));
                rbc.setFileshareDisallowed(post.containsKey(RobotsTxtConfig.FILESHARE));
                rbc.setHomepageDisallowed(post.containsKey(RobotsTxtConfig.HOMEPAGE));
                rbc.setLockedDisallowed(post.containsKey(RobotsTxtConfig.LOCKED));
                rbc.setNetworkDisallowed(post.containsKey(RobotsTxtConfig.NETWORK));
                rbc.setNewsDisallowed(post.containsKey(RobotsTxtConfig.NEWS));
                rbc.setStatusDisallowed(post.containsKey(RobotsTxtConfig.STATUS));
                rbc.setSurftipsDisallowed(post.containsKey(RobotsTxtConfig.SURFTIPS));
                rbc.setWikiDisallowed(post.containsKey(RobotsTxtConfig.WIKI));
                rbc.setProfileDisallowed(post.containsKey(RobotsTxtConfig.PROFILE));
                env.setConfig(SwitchboardConstants.ROBOTS_TXT, rbc.toString());
            }
        }
        
        prop.put(RobotsTxtConfig.ALL + ".checked", (rbc.isAllDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.BLOG + ".checked", (rbc.isBlogDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.BOOKMARKS + ".checked", (rbc.isBookmarksDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.DIRS + ".checked", (rbc.isDirsDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.FILESHARE + ".checked", (rbc.isFileshareDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.HOMEPAGE + ".checked", (rbc.isHomepageDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.LOCKED + ".checked", (rbc.isLockedDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.NETWORK + ".checked", (rbc.isNetworkDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.NEWS + ".checked", (rbc.isNewsDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.STATUS + ".checked", (rbc.isStatusDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.SURFTIPS + ".checked", (rbc.isSurftipsDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.WIKI + ".checked", (rbc.isWikiDisallowed()) ? "1" : "0");
        prop.put(RobotsTxtConfig.PROFILE + ".checked", (rbc.isProfileDisallowed()) ? "1" : "0");
        return prop;
    }

View Full Code Here

                else
                    serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));


                final ResponseHeader responseHeader = getDefaultHeaders(path);
                responseHeader.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"admin log-in\"");
                final servletProperties tp=new servletProperties();
                tp.put("returnto", path);
                HTTPDemon.sendRespondError(conProp, out, 5, 401, "Wrong Authentication", "", new File("proxymsg/authfail.inc"), tp, null, responseHeader);
                return;
            }


            // Authentication successful. remove brute-force flag
            serverCore.bfHost.remove(conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));


            // parse arguments
            serverObjects args = new serverObjects();
            int argc = 0;
            if (argsString == null) {
                // no args here, maybe a POST with multipart extension
                final int length = requestHeader.getContentLength();
                //System.out.println("HEADER: " + requestHeader.toString()); // DEBUG


                /* don't parse body in case of a POST CGI call since it has to be
                 * handed over to the CGI script unaltered and parsed by the script
                 */
                if (method.equals(HeaderFramework.METHOD_POST) &&
                        !(switchboard.getConfigBool("cgi.allow", false) &&
                        matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)))
                        ) {


                    // if its a POST, it can be either multipart or as args in the body
                    if ((requestHeader.containsKey(HeaderFramework.CONTENT_TYPE)) &&
                            (requestHeader.get(HeaderFramework.CONTENT_TYPE).toLowerCase().startsWith("multipart"))) {
                        // parse multipart
                        final Map<String, byte[]> files = HTTPDemon.parseMultipart(requestHeader, args, body);
                        // integrate these files into the args
                        if (files != null) {
                            final Iterator<Map.Entry<String, byte[]>> fit = files.entrySet().iterator();
                            Map.Entry<String, byte[]> entry;
                            while (fit.hasNext()) {
                                entry = fit.next();
                                args.put(entry.getKey() + "$file", entry.getValue());
                            }
                        }
                        argc = Integer.parseInt(requestHeader.get("ARGC"));
                    } else {
                        // parse args in body
                        argc = HTTPDemon.parseArgs(args, body, length);
                    }
                } else {
                    // no args
                    argsString = null;
                    args = null;
                    argc = 0;
                }
            } else {
                // simple args in URL (stuff after the "?")
                argc = HTTPDemon.parseArgs(args, argsString);
            }


            // check for cross site scripting - attacks in request arguments
            if (args != null && argc > 0) {
                // check all values for occurrences of script values
                final Iterator<String> e = args.values().iterator(); // enumeration of values
                String val;
                while (e.hasNext()) {
                    val = e.next();
                    if ((val != null) && (val.indexOf("<script",0) >= 0) && !path.equals("/Crawler_p.html")) {
                        // deny request
                        HTTPDemon.sendRespondError(conProp,out,4,403,null,"bad post values",null);
                        return;
                    }
                }
            }


            if (args != null) nocache = true;


            // we are finished with parsing
            // the result of value hand-over is in args and argc
            if (path.length() == 0) {
                HTTPDemon.sendRespondError(conProp,out,4,400,null,"Bad Request",null);
                out.flush();
                return;
            }
            File targetClass=null;


            // locate the file
            if (path.length() > 0 && path.charAt(0) != '/' && path.charAt(0) != '\\') path = "/" + path; // attach leading slash
            if (path.endsWith("index.html")) path = path.substring(0, path.length() - 10);


            // a different language can be desired (by i.e. ConfigBasic.html) than the one stored in the locale.language
            String localeSelection = switchboard.getConfig("locale.language","default");
            if (args != null && (args.containsKey("language"))) {
                // TODO 9.11.06 Bost: a class with information about available languages is needed.
                // the indexOf(".") is just a workaround because there from ConfigLanguage.html commes "de.lng" and
                // from ConfigBasic.html comes just "de" in the "language" parameter
                localeSelection = args.get("language", localeSelection);
                if (localeSelection.indexOf('.') != -1)
                    localeSelection = localeSelection.substring(0, localeSelection.indexOf('.'));
            }


            File targetFile = getLocalizedFile(path, localeSelection);
            String targetExt = (String) conProp.get("EXT"); if (targetExt == null) targetExt = "";
            targetClass = rewriteClassFile(new File(htDefaultPath, path));
            if (path.endsWith("/") || path.endsWith("\\")) {
                String testpath;
                // look for indexForward setting
                if (indexForward.length() > 0 && (targetFile = getOverlayedFile(path + indexForward)).exists()) {
                    testpath = path + indexForward;
                    targetClass = getOverlayedClass(testpath);
                    path = testpath;
                } else {
                    // attach default file name(s)
                    for (final String defaultFile : defaultFiles) {
                        testpath = path + defaultFile;
                        targetFile = getOverlayedFile(testpath);
                        targetClass = getOverlayedClass(testpath);
                        if (targetFile.exists()) {
                            path = testpath;
                            break;
                        }
                    }
                }
                targetFile = getLocalizedFile(path, localeSelection);


                //no defaultfile, send a dirlisting
                if (targetFile == null || !targetFile.exists() || (targetFile.exists() && targetFile.isDirectory())) {
                    final StringBuilder aBuffer = new StringBuilder();
                    aBuffer.append("<html>\n<head>\n</head>\n<body>\n<h1>Index of " + path + "</h1>\n  <ul>\n");
                    String[] list = targetFile.list();
                    if (list == null) list = new String[0]; // should not occur!
                    File f;
                    String size;
                    long sz;
                    String headline, author, description, publisher;
                    int images, links;
                    ContentScraper scraper;
                    for (final String element : list) {
                        f = new File(targetFile, element);
                        if (f.isDirectory()) {
                            aBuffer.append("    <li><a href=\"" + path + element + "/\">" + element + "/</a><br/></li>\n");
                        } else {
                            if (element.endsWith("html") || (element.endsWith("htm"))) {
                                scraper = ContentScraper.parseResource(f);
                                headline = scraper.getTitle();
                                author = scraper.getAuthor();
                                publisher = scraper.getPublisher();
                                description = scraper.getDescription();
                                images = scraper.getImages().size();
                                links = scraper.getAnchors().size();
                            } else {
                                headline = null;
                                author = null;
                                publisher = null;
                                description = null;
                                images = 0;
                                links = 0;
                            }
                            sz = f.length();
                            if (sz < 1024) {
                                size = sz + " bytes";
                            } else if (sz < 1024 * 1024) {
                                size = (sz / 1024) + " KB";
                            } else {
                                size = (sz / 1024 / 1024) + " MB";
                            }
                            aBuffer.append("    <li>");
                            if (headline != null && headline.length() > 0) aBuffer.append("<a href=\"" + element + "\"><b>" + headline + "</b></a><br/>");
                            aBuffer.append("<a href=\"" + path + element + "\">" + element + "</a><br/>");
                            if (author != null && author.length() > 0) aBuffer.append("Author: " + author + "<br/>");
                            if (publisher != null && publisher.length() > 0) aBuffer.append("Publisher: " + publisher + "<br/>");
                            if (description != null && description.length() > 0) aBuffer.append("Description: " + description + "<br/>");
                            aBuffer.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n");
                        }
                    }
                    aBuffer.append("  </ul>\n</body>\n</html>\n");


                    // write the list to the client
                    HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, "text/html; charset=UTF-8", aBuffer.length(), new Date(targetFile.lastModified()), null, new ResponseHeader(), null, null, true);
                    if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                        out.write(UTF8.getBytes(aBuffer.toString()));
                    }
                    return;
                }
            } else {
                    //XXX: you cannot share a .png/.gif file with a name like a class in htroot.
                    if ( !(targetFile.exists()) &&
                            !((path.endsWith("png")||path.endsWith("gif") ||
                            matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) ||
                            path.endsWith(".stream")) &&
                            targetClass!=null ) ){
                        targetFile = new File(htDocsPath, path);
                        targetClass = rewriteClassFile(new File(htDocsPath, path));
                    }
            }


            // implement proxy via url (not in servlet, because we need binary access on ouputStream)
            if (path.equals("/proxy.html")) {
              final List<Pattern> urlProxyAccess = Domains.makePatterns(sb.getConfig("proxyURL.access", "127.0.0.1"));
                final UserDB.Entry user = sb.userDB.getUser(requestHeader);
                final boolean user_may_see_proxyurl = Domains.matchesList(clientIP, urlProxyAccess) || (user!=null && user.hasRight(UserDB.AccessRight.PROXY_RIGHT));
              if (sb.getConfigBool("proxyURL", false) && user_may_see_proxyurl) {
                doURLProxy(args, conProp, requestHeader, out);
                return;
              }
              else {
              HTTPDemon.sendRespondError(conProp,out,3,403,"Access denied",null,null);
              }
            }


            // track all files that had been accessed so far
            if (targetFile != null && targetFile.exists()) {
                if (args != null && args.size() > 0) sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), path));
            }


            //File targetClass = rewriteClassFile(targetFile);
            //We need tp here
            servletProperties templatePatterns = null;
            Date targetDate;


            if ((targetClass != null) && (path.endsWith("png"))) {
                // call an image-servlet to produce an on-the-fly - generated image
                Object img = null;
                try {
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                    requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "png");
                    // in case that there are no args given, args = null or empty hashmap
                    img = invokeServlet(targetClass, requestHeader, args);
                } catch (final InvocationTargetException e) {
                    theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
                    e.getMessage() +
                    " target exception at " + targetClass + ": " +
                    e.getTargetException().toString() + ":" +
                    e.getTargetException().getMessage() +
                    "; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
                    Log.logException(e);
                    Log.logException(e.getTargetException());
                    targetClass = null;
                }
                if (img == null) {
                    // error with image generation; send file-not-found
                    HTTPDemon.sendRespondError(conProp, out, 3, 404, "File not Found", null, null);
                } else {
                    if (img instanceof RasterPlotter) {
                        final RasterPlotter yp = (RasterPlotter) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = Classification.ext2mime(targetExt, "text/html");
                        final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    if (img instanceof EncodedImage) {
                        final EncodedImage yp = (EncodedImage) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = Classification.ext2mime(targetExt, "text/html");
                        final ByteBuffer result = yp.getImage();


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    /*
                    if (img instanceof BufferedImage) {
                        final BufferedImage i = (BufferedImage) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = MimeTable.ext2mime(targetExt, "text/html");


                        // generate an byte array from the generated image
                        int width = i.getWidth(); if (width < 0) width = 96; // bad hack
                        int height = i.getHeight(); if (height < 0) height = 96; // bad hack
                        final ByteBuffer result = RasterPlotter.exportImage(i, targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                    */
                    if (img instanceof Image) {
                        final Image i = (Image) img;
                        // send an image to client
                        targetDate = new Date(System.currentTimeMillis());
                        nocache = true;
                        final String mimeType = Classification.ext2mime(targetExt, "text/html");


                        // generate an byte array from the generated image
                        int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
                        int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
                        final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
                        bi.createGraphics().drawImage(i, 0, 0, width, height, null);
                        final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt);


                        // write the array to the client
                        HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
                        if (!method.equals(HeaderFramework.METHOD_HEAD)) {
                            result.writeTo(out);
                        }
                    }
                }
            // old-school CGI execution
            } else if ((switchboard.getConfigBool("cgi.allow", false) // check if CGI execution is allowed in config
                    && matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) // "right" file extension?
                    && path.substring(0, path.indexOf(targetFile.getName())).toUpperCase().contains("/CGI-BIN/") // file in right directory?
                    && targetFile.exists())
                    ) {


                if (!targetFile.canExecute()) {
                    HTTPDemon.sendRespondError(
                            conProp,
                            out,
                            -1,
                            403,
                            null,
                            HeaderFramework.http1_1.get(
                                    Integer.toString(403)),
                            null);
                    Log.logWarning(
                            "HTTPD",
                            "CGI script " + targetFile.getPath()
                            + " could not be executed due to "
                            + "insufficient access rights.");
                } else {
                    String mimeType = "text/html";
                    int statusCode = 200;


                    final ProcessBuilder pb =
                            new ProcessBuilder(assembleCommandFromShebang(targetFile));
                    pb.directory(targetFile.getParentFile());


                    final String fileSeparator =
                            System.getProperty("file.separator", "/");


                    // set environment variables
                    final Map<String, String> env = pb.environment();
                    env.put(
                            "SERVER_SOFTWARE",
                            getDefaultHeaders(path).get(HeaderFramework.SERVER));
                    env.put("SERVER_NAME", sb.peers.mySeed().getName());
                    env.put("GATEWAY_INTERFACE", "CGI/1.1");
                    if (httpVersion != null) {
                        env.put("SERVER_PROTOCOL", httpVersion);
                    }
                    env.put("SERVER_PORT", switchboard.getConfig("port", "8090"));
                    env.put("REQUEST_METHOD", method);
    //                env.put("PATH_INFO", "");         // TODO: implement
    //                env.put("PATH_TRANSLATED", "");   // TODO: implement
                    env.put("SCRIPT_NAME", path);
                    if (argsString != null) {
                        env.put("QUERY_STRING", argsString);
                    }
                    env.put("REMOTE_ADDR", clientIP);
    //                env.put("AUTH_TYPE", "");         // TODO: implement
    //                env.put("REMOTE_USER", "");       // TODO: implement
    //                env.put("REMOTE_IDENT", "");      // I don't think we need this
                    env.put(
                            "DOCUMENT_ROOT",
                            switchboard.getAppPath().getAbsolutePath()
                            + fileSeparator + switchboard.getConfig("htDocsPath", "DATA/HTDOCS"));
                    if (requestHeader.getContentType() != null) {
                        env.put("CONTENT_TYPE", requestHeader.getContentType());
                    }
                    if (method.equalsIgnoreCase(HeaderFramework.METHOD_POST)
                            && body != null) {
                        env.put(
                                "CONTENT_LENGTH",
                                Integer.toString(requestHeader.getContentLength()));
                    }


                    /* add values from request header to environment
                     * (see: http://hoohoo.ncsa.uiuc.edu/cgi/env.html#headers) */
                    for (final Map.Entry<String, String> requestHeaderEntry
                            : requestHeader.entrySet()) {
                        env.put("HTTP_"
                            + requestHeaderEntry.getKey().toUpperCase().replace("-", "_"),
                            requestHeaderEntry.getValue());
                    }


                    int exitValue = 0;
                    String cgiBody = null;
                    final StringBuilder error = new StringBuilder(256);


                    try {
                        // start execution of script
                        final Process p = pb.start();


                        final OutputStream os =
                                new BufferedOutputStream(p.getOutputStream());


                        if (method.equalsIgnoreCase(
                                HeaderFramework.METHOD_POST) && body != null) {
                            final byte[] buffer = new byte[1024];
                            int len = requestHeader.getContentLength();
                            while (len > 0) {
                                body.read(buffer);
                                len = len - buffer.length;
                                os.write(buffer);
                            }
                        }


                        os.close();


                        try {
                            p.waitFor();
                        } catch (final InterruptedException ex) {


                        }


                        exitValue = p.exitValue();


                        final InputStream is =
                                new BufferedInputStream(p.getInputStream());


                        final InputStream es =
                                new BufferedInputStream(p.getErrorStream());


                        final StringBuilder processOutput =
                                new StringBuilder(1024);


                        while (is.available() > 0) {
                            processOutput.append((char) is.read());
                        }


                        while (es.available() > 0) {
                            error.append((char) es.read());
                        }


                        int indexOfDelimiter = processOutput.indexOf("\n\n", 0);
                        final String[] cgiHeader;
                        if (indexOfDelimiter > -1) {
                            cgiHeader =
                                    processOutput.substring(
                                            0, indexOfDelimiter).split("\n");
                        } else {
                            cgiHeader = new String[0];
                        }
                        cgiBody = processOutput.substring(indexOfDelimiter + 1);


                        String key;
                        String value;
                        for (final String element : cgiHeader) {
                            indexOfDelimiter = element.indexOf(':');
                            key = element.substring(0, indexOfDelimiter).trim();
                            value = element.substring(indexOfDelimiter + 1).trim();
                            conProp.put(key, value);
                            if ("Cache-Control".equals(key)
                                    && "no-cache".equals(value)) {
                                nocache = true;
                            } else if ("Content-type".equals(key)) {
                                mimeType = value;
                            } else if ("Status".equals(key)) {
                                if (key.length() > 2) {
                                    try {
                                        statusCode =
                                                Integer.parseInt(
                                                        value.substring(0, 3));
                                    } catch (final NumberFormatException ex) {
                                        Log.logWarning(
                                                "HTTPD",
                                                "CGI script " + targetFile.getPath()
                                                + " returned illegal status code \""
                                                + value + "\".");
                                    }
                                }
                            }
                        }
                    } catch (final IOException ex) {
                        exitValue = -1;
                    }


                    /* did the script return an exit value != 0
                     * and still there is supposed to be
                     * everything right with the HTTP status?
                     * -> change status to 500 since 200 would
                     * be a lie
                     */
                    if (exitValue != 0 && statusCode == 200) {
                        statusCode = 500;
                    }


                    targetDate = new Date(System.currentTimeMillis());


                    if (exitValue == 0
                            && cgiBody != null
                            && !cgiBody.isEmpty()) {
                        HTTPDemon.sendRespondHeader(
                                conProp,
                                out,
                                httpVersion,
                                statusCode,
                                null,
                                mimeType,
                                cgiBody.length(),
                                targetDate,
                                null,
                                null,
                                null,
                                null,
                                nocache);
                        out.write(UTF8.getBytes(cgiBody));
                    } else {
                        HTTPDemon.sendRespondError(
                                conProp,
                                out,
                                exitValue,
                                statusCode,
                                null,
                                HeaderFramework.http1_1.get(
                                        Integer.toString(statusCode)),
                                null);
                        Log.logWarning(
                                "HTTPD",
                                "CGI script " + targetFile.getPath()
                                + " returned exit value " + exitValue
                                + ", body empty: "
                                + (cgiBody == null || cgiBody.isEmpty()));
                        if (error.length() > 0) {
                            Log.logWarning("HTTPD", "Reported error: " + error);
                        }
                    }
                }
            } else if ((targetClass != null) && (path.endsWith(".stream"))) {
                // call rewrite-class
                requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "stream");
                //requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
                //requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);


                HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null);


                // in case that there are no args given, args = null or empty hashmap
                /* servletProperties tp = (servlerObjects) */ invokeServlet(targetClass, requestHeader, args);
                forceConnectionClose(conProp);
                return;
            } else if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
                // we have found a file that can be written to the client
                // if this file uses templates, then we use the template
                // re-write - method to create an result
                String mimeType = Classification.ext2mime(targetExt, "text/html");
                String ext = (String) conProp.get("EXT"); if (ext == null) ext = "";
                final boolean zipContent = requestHeader.acceptGzip() && HTTPDemon.shallTransportZipped("." + ext);
                if (path.endsWith("html") ||
                        path.endsWith("htm") ||
                        path.endsWith("xml") ||
                        path.endsWith("json") ||
                        path.endsWith("rdf") ||
                        path.endsWith("rss") ||
                        path.endsWith("csv") ||
                        path.endsWith("pac") ||
                        path.endsWith("src") ||
                        path.endsWith("vcf") ||
                        path.endsWith("kml") ||
                        path.endsWith("gpx") ||
                        path.endsWith("css") ||
                        path.endsWith("/") ||
                        path.equals("/robots.txt")) {


                    /*targetFile = getLocalizedFile(path);
                    if (!(targetFile.exists())) {
                        // try to find that file in the htDocsPath
                        File trialFile = new File(htDocsPath, path);
                        if (trialFile.exists()) targetFile = trialFile;
                    }*/




                    // call rewrite-class


                    if (targetClass != null) {
                        // CGI-class: call the class to create a property for rewriting
                        try {
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
                            final int ep = path.lastIndexOf(".");
                            requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, path.substring(ep + 1));
                            // in case that there are no args given, args = null or empty hashmap
                            final Object tmp = invokeServlet(targetClass, requestHeader, args);
                            if (tmp == null) {
                                // if no args given, then tp will be an empty Hashtable object (not null)
                                templatePatterns = new servletProperties();
                            } else if (tmp instanceof servletProperties) {
                                templatePatterns = (servletProperties) tmp;
                            } else {
                                templatePatterns = new servletProperties((serverObjects) tmp);
                            }
                            // check if the servlets requests authentication
                            if (templatePatterns.containsKey(servletProperties.ACTION_AUTHENTICATE)) {
                                // handle brute-force protection
                                if (realmProp != null) {
                                    Log.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
                                    final Integer attempts = serverCore.bfHost.get(clientIP);
                                    if (attempts == null)
                                        serverCore.bfHost.put(clientIP, Integer.valueOf(1));
                                    else
                                        serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
                                }
                                // send authentication request to browser
                                final ResponseHeader headers = getDefaultHeaders(path);
                                headers.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"" + templatePatterns.get(servletProperties.ACTION_AUTHENTICATE, "") + "\"");
                                HTTPDemon.sendRespondHeader(conProp,out,httpVersion,401,headers);
                                return;
                            } else if (templatePatterns.containsKey(servletProperties.ACTION_LOCATION)) {
                                String location = templatePatterns.get(servletProperties.ACTION_LOCATION, "");
                                if (location.length() == 0) location = path;


                                final ResponseHeader headers = getDefaultHeaders(path);
                                headers.setAdditionalHeaderProperties(templatePatterns.getOutgoingHeader().getAdditionalHeaderProperties()); //put the cookies into the new header TODO: can we put all headerlines, without trouble?
                                headers.put(HeaderFramework.LOCATION,location);
                                HTTPDemon.sendRespondHeader(conProp,out,httpVersion,302,headers);
                                return;
                            }
                            // add the application version, the uptime and the client name to every rewrite table
                            templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
                            templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() -  serverCore.startupTime) / 1000) / 60); // uptime in minutes
                            templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
                            templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID());
                            templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
                            final Seed myPeer = sb.peers.mySeed();
                            templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
                            templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
                            //System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug
                        } catch (final InvocationTargetException e) {
                            if (e.getCause() instanceof InterruptedException) {
                                throw new InterruptedException(e.getCause().getMessage());
                            }


                            theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
                                    e.getMessage() +
                                    " target exception at " + targetClass + ": " +
                                    e.getTargetException().toString() + ":" +
                                    e.getTargetException().getMessage(),e);
                            targetClass = null;
                            throw e;
                        }
                        nocache = true;
                    }


                    targetDate = new Date(targetFile.lastModified());
                    Date expireDate = null;
                    if (templatePatterns == null) {
                      // if the file will not be changed, cache it in the browser
                      expireDate = new Date(new Date().getTime() + (31l * 24 * 60 * 60 * 1000));
                    }




                    // rewrite the file
                    InputStream fis = null;


                    // read the file/template
                    TemplateCacheEntry templateCacheEntry = null;
                    final long fileSize = targetFile.length();
                    if (useTemplateCache && fileSize <= 512 * 1024) {
                        // read from cache
                        SoftReference<TemplateCacheEntry> ref = templateCache.get(targetFile);
                        if (ref != null) {
                            templateCacheEntry = ref.get();
                            if (templateCacheEntry == null) templateCache.remove(targetFile);
                        }


                        final Date targetFileDate = new Date(targetFile.lastModified());
                        if (templateCacheEntry == null || targetFileDate.after(templateCacheEntry.lastModified)) {
                            // loading the content of the template file into
                            // a byte array
                            templateCacheEntry = new TemplateCacheEntry();
                            templateCacheEntry.lastModified = targetFileDate;
                            templateCacheEntry.content = FileUtils.read(targetFile);


                            // storing the content into the cache
                            ref = new SoftReference<TemplateCacheEntry>(templateCacheEntry);
                            if (MemoryControl.shortStatus()) templateCache.clear();
                            templateCache.put(targetFile, ref);
                            if (theLogger.isFinest()) theLogger.logFinest("Cache MISS for file " + targetFile);
                        } else {
                            if (theLogger.isFinest()) theLogger.logFinest("Cache HIT for file " + targetFile);
                        }


                        // creating an inputstream needed by the template
                        // rewrite function
                        fis = new ByteArrayInputStream(templateCacheEntry.content);
                        templateCacheEntry = null;
                    } else if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
                        // read file completely into ram, avoid that too many files are open at the same time
                        fis = new ByteArrayInputStream(FileUtils.read(targetFile));
                    } else {
                        fis = new BufferedInputStream(new FileInputStream(targetFile));
                    }


                    if (mimeType.startsWith("text")) {
                        // every text-file distributed by yacy is UTF-8
                        if(!path.startsWith("/repository")) {
                            mimeType = mimeType + "; charset=UTF-8";
                        } else {
                            // detect charset of html-files
                            if((path.endsWith("html") || path.endsWith("htm"))) {
                                // save position
                                fis.mark(1000);
                                // scrape document to look up charset
                                final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
                                final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
                                if(charset != null)
                                    mimeType = mimeType + "; charset="+charset;
                                // reset position
                                fis.reset();
                            }
                        }
                    }


                    // write the array to the client
                    // we can do that either in standard mode (whole thing completely) or in chunked mode
                    // since yacy clients do not understand chunked mode (yet), we use this only for communication with the administrator
                    final boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
                    final boolean chunked = !method.equals(HeaderFramework.METHOD_HEAD) && !yacyClient && httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1);
                    if (chunked) {
                        // send page in chunks and parse SSIs
                        final ByteBuffer o = new ByteBuffer();
                        // apply templates
                        TemplateEngine.writeTemplate(fis, o, templatePatterns, UNRESOLVED_PATTERN);
                        fis.close();
                        HTTPDemon.sendRespondHeader(conProp, out,
                                httpVersion, 200, null, mimeType, -1,
                                targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                null, "chunked", nocache);
                        // send the content in chunked parts, see RFC 2616 section 3.6.1
                        final ChunkedOutputStream chos = new ChunkedOutputStream(out);
                        // GZIPOutputStream does not implement flush (this is a bug IMHO)
                        // so we can't compress this stuff, without loosing the cool SSI trickle feature
                        ServerSideIncludes.writeSSI(o, chos, realmProp, clientIP, requestHeader);
                        //chos.write(result);
                        chos.finish();
                    } else {
                        // send page as whole thing, SSIs are not possible
                        final String contentEncoding = (zipContent) ? "gzip" : null;
                        // apply templates
                        final ByteBuffer o1 = new ByteBuffer();
                        TemplateEngine.writeTemplate(fis, o1, templatePatterns, ASCII.getBytes("-UNRESOLVED_PATTERN-"));
                        fis.close();
                        final ByteBuffer o = new ByteBuffer();


                        if (zipContent) {
                            GZIPOutputStream zippedOut = new GZIPOutputStream(o);
                            ServerSideIncludes.writeSSI(o1, zippedOut, realmProp, clientIP, requestHeader);
                            //httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
                            zippedOut.finish();
                            zippedOut.flush();
                            zippedOut.close();
                            zippedOut = null;
                        } else {
                            ServerSideIncludes.writeSSI(o1, o, realmProp, clientIP, requestHeader);
                            //httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
                        }
                        if (method.equals(HeaderFramework.METHOD_HEAD)) {
                            HTTPDemon.sendRespondHeader(conProp, out,
                                    httpVersion, 200, null, mimeType, o.length(),
                                    targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                    contentEncoding, null, nocache);
                        } else {
                            final byte[] result = o.getBytes(); // this interrupts streaming (bad idea!)
                            HTTPDemon.sendRespondHeader(conProp, out,
                                    httpVersion, 200, null, mimeType, result.length,
                                    targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
                                    contentEncoding, null, nocache);
                            FileUtils.copy(result, out);
                        }
                    }
                } else { // no html

View Full Code Here

0 1 2

TOP

Related Classes of de.anomic.server.servletProperties

de.anomic.http.server.HTTPDFileHandler

Help

IndexCreateWWWRemoteQueue_p

net.yacy.cora.protocol.ResponseHeader

robots

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.