Package de.anomic.server

Examples of de.anomic.server.serverObjects


public class Load_PHPBB3 {
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
       
        // define visible variables
        String a = sb.peers.mySeed().getPublicAddress();
        if (a == null) a = "localhost:" + sb.getConfig("port", "8090");
        final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");
        final String repository = "http://" + a + "/repository/";
        prop.put("starturl", (intranet) ? repository : "http://");
        prop.put("address", a);
       
        // return rewrite properties
        return prop;
    }
View Full Code Here


   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {
       
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        final String langPath = env.getDataPath("locale.work", "DATA/LOCALE/locales").getAbsolutePath();
        String lang = env.getConfig("locale.language", "default");
       
        final int authentication = sb.adminAuthenticated(header);
        if (authentication < 2) {
            // must authenticate
            prop.put("AUTHENTICATE", "admin log-in");
            return prop;
        }
       
        // store this call as api call
        if (post != null && post.containsKey("set")) {
            sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");
        }
       
        //boolean doPeerPing = false;
        if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
            InstantBusyThread.oneTimeJob(sb.yc, "peerPing", null, 0);
            //doPeerPing = true;
        }
       
        // language settings
        if (post != null && post.containsKey("language"&& !lang.equals(post.get("language", "default")) &&
                (Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
            prop.put("changedLanguage", "1");
        }
       
        // peer name settings
        final String peerName = (post == null) ? sb.peers.mySeed().getName() : post.get("peername", "");
       
        // port settings
        final long port;
        if (post != null && post.getInt("port", 0) > 1023) {
            port = post.getLong("port", 8090);
        } else {
            port = env.getConfigLong("port", 8090); //this allows a low port, but it will only get one, if the user edits the config himself.
        }

        // check if peer name already exists
        final yacySeed oldSeed = sb.peers.lookupByName(peerName);
        if (oldSeed == null &&
            !peerName.equals(sb.peers.mySeed().getName()) &&
            Pattern.compile("[A-Za-z0-9\\-_]{3,80}").matcher(peerName).matches()) {
            sb.peers.setMyName(peerName);
            sb.peers.saveMySeed();
        }
       
        // UPnP config
        final boolean upnp;
        if (post != null && post.containsKey("port")) { // hack to allow checkbox
            upnp = post.containsKey("enableUpnp");
            if (upnp && !sb.getConfigBool(SwitchboardConstants.UPNP_ENABLED, false)) {
                UPnP.addPortMapping();
            }
            sb.setConfig(SwitchboardConstants.UPNP_ENABLED, upnp);
            if (!upnp) {
                UPnP.deletePortMapping();
            }
        } else {
            upnp = false;
        }
       
        // check port
        final boolean reconnect;
        if (!(env.getConfigLong("port", port) == port)) {
            // validate port
            final serverCore theServerCore = (serverCore) env.getThread("10_httpd");
            env.setConfig("port", port);
           
            // redirect the browser to the new port
            reconnect = true;
           
            // renew upnp port mapping
            if (upnp) {
                UPnP.addPortMapping();
            }
           
            String host = null;
            if (header.containsKey(HeaderFramework.HOST)) {
                host = header.get(HeaderFramework.HOST);
                final int idx = host.indexOf(':');
                if (idx != -1) host = host.substring(0,idx);
            } else {
                host = Domains.myPublicLocalIP().getHostAddress();
            }
           
            prop.put("reconnect", "1");
            prop.put("reconnect_host", host);
            prop.put("nextStep_host", host);
            prop.put("reconnect_port", port);
            prop.put("nextStep_port", port);
            prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");
            prop.put("nextStep_sslSupport", theServerCore.withSSL() ? "1" : "0");
           
            // generate new shortcut (used for Windows)
            //yacyAccessible.setNewPortBat(Integer.parseInt(port));
            //yacyAccessible.setNewPortLink(Integer.parseInt(port));
           
            // force reconnection in 7 seconds
            theServerCore.reconnect(7000);
        } else {
            reconnect = false;
            prop.put("reconnect", "0");
        }

        // set a use case
        String networkName = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "");
        if (post != null && post.containsKey("usecase")) {
            if ("freeworld".equals(post.get("usecase", "")) && !"freeworld".equals(networkName)) {
                // switch to freeworld network
                sb.switchNetwork("defaults/yacy.network.freeworld.unit");
                // switch to p2p mode
                sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, true);
                sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
            }
            if ("portal".equals(post.get("usecase", "")) && !"webportal".equals(networkName)) {
                // switch to webportal network
                sb.switchNetwork("defaults/yacy.network.webportal.unit");
                // switch to robinson mode
                sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
                sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
            }
            if ("intranet".equals(post.get("usecase", "")) && !"intranet".equals(networkName)) {
                // switch to intranet network
                sb.switchNetwork("defaults/yacy.network.intranet.unit");
                // switch to p2p mode: enable ad-hoc networks between intranet users
                sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
                sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
            }
            if ("intranet".equals(post.get("usecase", ""))) {
                final String repositoryPath = post.get("repositoryPath", "/DATA/HTROOT/repository");
                final File repository = ((repositoryPath.length() > 0 && repositoryPath.charAt(0) == '/') || (repositoryPath.length() > 1 && repositoryPath.charAt(1) == ':')) ? new File(repositoryPath) : new File(sb.getDataPath(), repositoryPath);
                if (repository.exists() && repository.isDirectory()) {
                  sb.setConfig("repositoryPath", repositoryPath);
                }
            }
        }
       
        networkName = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "");
        if ("freeworld".equals(networkName)) {
            prop.put("setUseCase", 1);
            prop.put("setUseCase_freeworldChecked", 1);
        } else if ("webportal".equals(networkName)) {
            prop.put("setUseCase", 1);
            prop.put("setUseCase_portalChecked", 1);
        } else if ("intranet".equals(networkName)) {
            prop.put("setUseCase", 1);
            prop.put("setUseCase_intranetChecked", 1);
        } else {
            prop.put("setUseCase", 0);
        }
        prop.put("setUseCase_port", port);
        prop.put("setUseCase_repositoryPath", sb.getConfig("repositoryPath", "/DATA/HTROOT/repository"));
       
        // check if values are proper
        final boolean properPassword = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").length() > 0) || sb.getConfigBool("adminAccountForLocalhost", false);
        final boolean properName = (sb.peers.mySeed().getName().length() >= 3) && (!(yacySeed.isDefaultPeerName(sb.peers.mySeed().getName())));
        final boolean properPort = (sb.peers.mySeed().isSenior()) || (sb.peers.mySeed().isPrincipal());
       
        if ((env.getConfig("defaultFiles", "").startsWith("ConfigBasic.html,"))) {
            env.setConfig("defaultFiles", env.getConfig("defaultFiles", "").substring(17));
            env.setConfig("browserPopUpPage", "Status.html");
            HTTPDFileHandler.initDefaultPath();
        }
       
        prop.put("statusName", properName ? "1" : "0");
        prop.put("statusPort", properPort ? "1" : "0");
        if (reconnect) {
            prop.put("nextStep", NEXTSTEP_RECONNECT);
        } else if (!properName) {
            prop.put("nextStep", NEXTSTEP_PEERNAME);
        } else if (!properPassword) {
            prop.put("nextStep", NEXTSTEP_PWD);
        } else if (!properPort) {
            prop.put("nextStep", NEXTSTEP_PEERPORT);
        } else {
            prop.put("nextStep", NEXTSTEP_FINISHED);
        }
               
        final boolean upnp_enabled = env.getConfigBool(SwitchboardConstants.UPNP_ENABLED, false);
        prop.put("upnp", "1");
        prop.put("upnp_enabled", upnp_enabled ? "1" : "0");
        if (upnp_enabled) {
            prop.put("upnp_success", (UPnP.getMappedPort() > 0) ? "2" : "1");
        }
        else {
            prop.put("upnp_success", "0");
        }
       
        // set default values      
        prop.putHTML("defaultName", sb.peers.mySeed().getName());
        prop.putHTML("defaultPort", env.getConfig("port", "8090"));
        lang = env.getConfig("locale.language", "default"); // re-assign lang, may have changed
        if ("default".equals(lang)) {
            prop.put("langDeutsch", "0");
            prop.put("langFrancais", "0");
            prop.put("langEnglish", "1");
        } else if ("fr".equals(lang)) {
            prop.put("langDeutsch", "0");
            prop.put("langFrancais", "1");
            prop.put("langEnglish", "0");
        } else if ("de".equals(lang)) {
            prop.put("langDeutsch", "1");
            prop.put("langFrancais", "0");
            prop.put("langEnglish", "0");
        } else {
            prop.put("langDeutsch", "0");
            prop.put("langFrancais", "0");
            prop.put("langEnglish", "0");
        }
        return prop;
    }
View Full Code Here

import de.anomic.server.serverSwitch;

public class ViewLog_p {
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final serverObjects prop = new serverObjects();
        String[] log = new String[0];
        boolean reversed = false;
        boolean json = false;
        int maxlines = 400, lines = 200;
        /* Usually a regex like this would make no sense, ".*" would be
         * sufficient, but ".*.*" makes it a little bit more convenient
         * for the user to input regexes like ".*FOO.*" in the HTML
         * interface.
         */
        String filter = ".*.*";
       
        if(post != null){
            reversed = (post.containsKey("mode") && "reversed".equals(post.get("mode")));
            json = post.containsKey("json");

            if(post.containsKey("lines")){
                lines = post.getInt("lines", lines);
            }

            if(post.containsKey("filter")){
                filter = post.get("filter");
            }
        }

        final Logger logger = Logger.getLogger("");
        final Handler[] handlers = logger.getHandlers();
        boolean displaySubmenu = false;
        for (final Handler handler : handlers) {
            if (handler instanceof GuiHandler) {
                maxlines = ((GuiHandler)handler).getSize();
                if (lines > maxlines) lines = maxlines;
                log = ((GuiHandler)handler).getLogLines(reversed,lines);
            } else if (handler instanceof LogalizerHandler) {
                displaySubmenu = true;
             }
        }
       
        prop.put("submenu", displaySubmenu ? "1" : "0");
        prop.put("reverseChecked", reversed ? "1" : "0");
        prop.put("lines", lines);
        prop.put("maxlines",maxlines);
        prop.putHTML("filter", filter);
       
        // trying to compile the regular expression filter expression
        Matcher filterMatcher = null;
        try {
            final Pattern filterPattern = Pattern.compile(filter,Pattern.MULTILINE);
            filterMatcher = filterPattern.matcher("");
        } catch (final PatternSyntaxException e) {
            Log.logException(e);
        }

        int level = 0;
        int lc = 0;
        for (final String logLine : log) {
            final String nextLogLine = logLine.trim();

            if (filterMatcher != null) {
              filterMatcher.reset(nextLogLine);
              if (!filterMatcher.find()) continue;
            }

            if (nextLogLine.startsWith("E ")) {
                level = 4;
            } else if (nextLogLine.startsWith("W ")) {
                level = 3;
            } else if (nextLogLine.startsWith("S ")) {
                level = 2;
            } else if (nextLogLine.startsWith("I ")) {
                level = 1;
            } else if (nextLogLine.startsWith("D ")) {
                level = 0;
            }

            prop.put("log_" + lc + "_level", level);

            if (json) {
                prop.putJSON("log_" + lc + "_line", nextLogLine);
            } else {
                prop.putHTML("log_" + lc + "_line", nextLogLine);
            }

            lc++;
        }
        prop.put("log", lc);
       
        // return rewrite properties
        return prop;
    }
View Full Code Here

public class ConfigNetwork_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {
       
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        int commit = 0;
       
        // load all options for network definitions
        final File networkBootstrapLocationsFile = new File(new File(sb.getAppPath(), "defaults"), "yacy.networks");
        final Set<String> networkBootstrapLocations = FileUtils.loadList(networkBootstrapLocationsFile);
       
        if (post != null) {
           
            // store this call as api call
            sb.tables.recordAPICall(post, "ConfigNetwork_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "network settings");
           
            if (post.containsKey("changeNetwork")) {
                final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
                if (networkDefinition.equals(sb.getConfig("network.unit.definition", ""))) {
                    // no change
                    commit = 3;
                } else {
                    // shut down old network and index, start up new network and index
                    commit = 1;
                    sb.switchNetwork(networkDefinition);
                }
            }
           
            if (post.containsKey("save")) {
               
                // DHT control
                boolean indexDistribute = "on".equals(post.get("indexDistribute", ""));
                boolean indexReceive = "on".equals(post.get("indexReceive", ""));
                final boolean robinsonmode = "robinson".equals(post.get("network", ""));
                if (robinsonmode) {
                    indexDistribute = false;
                    indexReceive = false;
                    commit = 1;
                } else {
                    if (!indexDistribute && !indexReceive) {
                        prop.put("commitDHTIsRobinson", "1");
                        commit = 2;
                    } else if (indexDistribute && indexReceive) {
                        commit = 1;
                    } else {
                        if (!indexReceive) {
                            prop.put("commitDHTNoGlobalSearch", "1");
                        }
                        commit = 1;
                    }
                }
               
                if (indexDistribute) {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, true);
                } else {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
                }
   
                if ("on".equals(post.get("indexDistributeWhileCrawling",""))) {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true);
                } else {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, false);
                }
   
                if ("on".equals(post.get("indexDistributeWhileIndexing",""))) {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true);
                } else {
                    sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, false);
                }
   
                if (indexReceive) {
                    sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
                    sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
                } else {
                    sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
                    sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
                    sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false);
                }
   
                if ("on".equals(post.get("indexReceiveBlockBlacklist", ""))) {
                    sb.setConfig("indexReceiveBlockBlacklist", true);
                } else {
                    sb.setConfig("indexReceiveBlockBlacklist", false);
                }
                   
                if (post.containsKey("peertags")) {
                    sb.peers.mySeed().setPeerTags(MapTools.string2set(normalizedList(post.get("peertags")), ","));
                }
               
                sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer"));
                sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", "")));
                sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", "")));
               
                // update the cluster hash set
                sb.clusterhashes = sb.peers.clusterHashes(sb.getConfig("cluster.peers.yacydomain", ""));
            }           
        }
       
        // write answer code
        prop.put("commit", commit);
       
        // write remote crawl request settings
        prop.put("crawlResponse", sb.getConfigBool("crawlResponse", false) ? "1" : "0");
        final long RTCbusySleep = Math.max(1, env.getConfigInt(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));
        final int RTCppm = (int) (60000L / RTCbusySleep);
        prop.put("acceptCrawlLimit", RTCppm);
       
        final boolean indexDistribute = sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW, true);
        final boolean indexReceive = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
        prop.put("indexDistributeChecked", (indexDistribute) ? "1" : "0");
        prop.put("indexDistributeWhileCrawling.on", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "1" : "0");
        prop.put("indexDistributeWhileCrawling.off", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_CRAWLING, true)) ? "0" : "1");
        prop.put("indexDistributeWhileIndexing.on", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "1" : "0");
        prop.put("indexDistributeWhileIndexing.off", (sb.getConfigBool(SwitchboardConstants.INDEX_DIST_ALLOW_WHILE_INDEXING, true)) ? "0" : "1");
        prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0");
        prop.put("indexReceiveBlockBlacklistChecked.on", (sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "1" : "0");
        prop.put("indexReceiveBlockBlacklistChecked.off", (sb.getConfigBool("indexReceiveBlockBlacklist", true)) ? "0" : "1");
        prop.putHTML("peertags", MapTools.set2string(sb.peers.mySeed().getPeerTags(), ",", false));

        // set seed information directly
        sb.peers.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false));
        sb.peers.mySeed().setFlagAcceptRemoteIndex(indexReceive);
       
        // set p2p/robinson mode flags and values
        prop.put("p2p.checked", (indexDistribute || indexReceive) ? "1" : "0");
        prop.put("robinson.checked", (indexDistribute || indexReceive) ? "0" : "1");
        prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", ""));
        prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", ""));
        StringBuilder hashes = new StringBuilder();
        for (final byte[] h : sb.clusterhashes.keySet()) {
            hashes.append(", ").append(ASCII.String(h));
        }
        if (hashes.length() > 2) {
            hashes = hashes.delete(0, 2);
        }
       
        prop.put("cluster.peers.yacydomain.hashes", hashes.toString());
       
        // set p2p mode flags
        prop.put("privatepeerChecked", ("privatepeer".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0");
        prop.put("privateclusterChecked", ("privatecluster".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0");
        prop.put("publicclusterChecked", ("publiccluster".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0");
        prop.put("publicpeerChecked", ("publicpeer".equals(sb.getConfig("cluster.mode", ""))) ? "1" : "0");
       
        // set network configuration
        prop.putHTML("network.unit.definition", sb.getConfig("network.unit.definition", ""));
        prop.putHTML("network.unit.name", sb.getConfig(SwitchboardConstants.NETWORK_NAME, ""));
        prop.putHTML("network.unit.description", sb.getConfig("network.unit.description", ""));
        prop.putHTML("network.unit.domain", sb.getConfig(SwitchboardConstants.NETWORK_DOMAIN, ""));
        prop.putHTML("network.unit.dht", sb.getConfig("network.unit.dht", ""));
        networkBootstrapLocations.remove(sb.getConfig("network.unit.definition", ""));
        int c = 0;
        for (final String s: networkBootstrapLocations) {
            prop.put("networks_" + c++ + "_network", s);
        }
        prop.put("networks", c);
       
        return prop;
    }
View Full Code Here

public class getpageinfo_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();

        // avoid UNRESOLVED PATTERN
        prop.put("title", "");
        prop.put("desc", "");
        prop.put("lang", "");
        prop.put("robots-allowed", "3"); //unknown
        prop.put("sitemap", "");
        prop.put("favicon","");
        prop.put("sitelist", "");
        prop.put("filter", ".*");

        // default actions
        String actions="title,robots";

        if (post != null && post.containsKey("url")) {
            if(post.containsKey("actions"))
                actions=post.get("actions");
            String url=post.get("url");
      if(url.toLowerCase().startsWith("ftp://")){
        prop.put("robots-allowed", "1");
        prop.putXML("title", "FTP: "+url);
                return prop;
      } else if (!url.startsWith("http://") &&
                   !url.startsWith("https://") &&
                   !url.startsWith("ftp://") &&
                   !url.startsWith("smb://") &&
                  !url.startsWith("file://")) {
                url = "http://" + url;
            }
            if (actions.indexOf("title")>=0) {
                DigestURI u = null;
                try {
                    u = new DigestURI(url);
                } catch (final MalformedURLException e) {
                    // fail, do nothing
                }
                ContentScraper scraper = null;
                if (u != null) try {
                    scraper = sb.loader.parseResource(u, CacheStrategy.IFEXIST);
                } catch (final IOException e) {
                    // now thats a fail, do nothing
                }
                if (scraper != null) {
                    // put the document title
                    prop.putXML("title", scraper.getTitle());

                    // put the favicon that belongs to the document
                    prop.put("favicon", (scraper.getFavicon()==null) ? "" : scraper.getFavicon().toString());

                    // put keywords
                    final String list[]=scraper.getKeywords();
                    int count = 0;
                    for (final String element : list) {
                        final String tag = element;
                        if (!tag.equals("")) {
                            prop.putXML("tags_"+count+"_tag", tag);
                            count++;
                        }
                    }
                    prop.put("tags", count);
                    // put description
                    prop.putXML("desc", scraper.getDescription());
                    // put language
                    final Set<String> languages = scraper.getContentLanguages();
                    prop.putXML("lang", (languages == null) ? "unknown" : languages.iterator().next());

                    // get links and put them into a semicolon-separated list
                    final Set<MultiProtocolURI> uris = scraper.getAnchors().keySet();
                    final StringBuilder links = new StringBuilder(uris.size() * 80);
                    final StringBuilder filter = new StringBuilder(uris.size() * 40);
                    count = 0;
                    for (final MultiProtocolURI uri: uris) {
                        links.append(';').append(uri.toNormalform(true, false));
                        filter.append('|').append(uri.getProtocol()).append("://").append(uri.getHost()).append(".*");
                        prop.putXML("links_" + count + "_link", uri.toNormalform(true, false));
                        count++;
                    }
                    prop.put("links", count);
                    prop.putXML("sitelist", links.length() > 0 ? links.substring(1) : "");
                    prop.putXML("filter", filter.length() > 0 ? filter.substring(1) : ".*");
                }
            }
            if (actions.indexOf("robots")>=0) {
                try {
                    final DigestURI theURL = new DigestURI(url);

                  // determine if crawling of the current URL is allowed
                    RobotsTxtEntry robotsEntry;
                    try {
                        robotsEntry = sb.robots.getEntry(theURL, sb.peers.myBotIDs());
                    } catch (final IOException e) {
                        robotsEntry = null;
                    }
                  prop.put("robots-allowed", robotsEntry == null ? 1 : robotsEntry.isDisallowed(theURL) ? 0 : 1);

                    // get the sitemap URL of the domain
                    final MultiProtocolURI sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap();
                    prop.putXML("sitemap", sitemapURL == null ? "" : sitemapURL.toString());
                } catch (final MalformedURLException e) {}
            }

        }
        // return rewrite properties
View Full Code Here

public class ynetSearch {
 
  public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {       
        final Switchboard switchboard = (Switchboard) env;
        final boolean isAdmin=switchboard.verifyAuthentication(header, true);
        final serverObjects prop = new serverObjects();             
               
      if(post != null){       
        if(!isAdmin){
      // force authentication if desired
          if(post.containsKey("login")){
            prop.put("AUTHENTICATE","admin log-in");
          }
          return prop;
        } else {
          InputStream is = null;          
          try {
              String searchaddress = post.get("url");
              if (!searchaddress.startsWith("http://")) {
                  // a relative path .. this addresses the local peer
                  searchaddress = "http://" + switchboard.peers.mySeed().getPublicAddress() + ((searchaddress.length() > 0 && searchaddress.charAt(0) == '/') ? "" : "/") + searchaddress;
              }
              post.remove("url");
              post.remove("login");
              final Iterator <Map.Entry<String, String>> it = post.entrySet().iterator();
              String s = searchaddress;
              Map.Entry<String, String> k;
              while(it.hasNext()) {
                k = it.next();
                s = s + "&" + k.getKey() + "=" + k.getValue();               
              }
            // final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord");                      
            final URL url = new URL(s);            
            is = url.openStream();
            final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next();           
            prop.put("http", httpout);
          }
          catch ( final Exception e ) {
            prop.put("url", "error!");
          }
          finally {
            if ( is != null )
              try { is.close(); } catch ( final IOException e ) { }
          }
View Full Code Here

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        // inital values for AJAX Elements (without JavaScript)
        final serverObjects prop = new serverObjects();
        prop.put("rejected", 0);
        prop.put("urlpublictextSize", 0);
        prop.put("rwipublictextSize", 0);
        prop.put("list", "0");
        prop.put("loaderSize", 0);
        prop.put("loaderMax", 0);
        prop.put("list-loader", 0);
        prop.put("localCrawlSize", sb.crawlQueues.coreCrawlJobSize());
        prop.put("localCrawlState", "");
        prop.put("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
        prop.put("limitCrawlState", "");
        prop.put("remoteCrawlSize", sb.crawlQueues.limitCrawlJobSize());
        prop.put("remoteCrawlState", "");
        prop.put("list-remote", 0);
        prop.put("forwardToCrawlStart", "0");

        // get segment
        Segment indexSegment = null;
        if (post != null && post.containsKey("segment")) {
            final String segmentName = post.get("segment");
            if (sb.indexSegments.segmentExist(segmentName)) {
                indexSegment = sb.indexSegments.segment(segmentName);
            }
        } else {
            // take default segment
            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }

        prop.put("info", "0");

        if (post != null && post.containsKey("continue")) {
            // continue queue
            final String queue = post.get("continue", "");
            if ("localcrawler".equals(queue)) {
                sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
            } else if ("remotecrawler".equals(queue)) {
                sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
            }
        }

        if (post != null && post.containsKey("pause")) {
            // pause queue
            final String queue = post.get("pause", "");
            if ("localcrawler".equals(queue)) {
                sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
            } else if ("remotecrawler".equals(queue)) {
                sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
            }
        }

        if (post != null && post.containsKey("crawlingstart")) {
            // init crawl
            if (sb.peers == null) {
                prop.put("info", "3");
            } else {
                String crawlingStart = post.get("crawlingURL","").trim(); // the crawljob start url
                // add the prefix http:// if necessary
                int pos = crawlingStart.indexOf("://");
                if (pos == -1) {
                    if (crawlingStart.startsWith("www")) crawlingStart = "http://" + crawlingStart;
                    if (crawlingStart.startsWith("ftp")) crawlingStart = "ftp://" + crawlingStart;
                }

                // remove crawlingFileContent before we record the call
                final String crawlingFileName = post.get("crawlingFile");
                final File crawlingFile = (crawlingFileName != null && crawlingFileName.length() > 0) ? new File(crawlingFileName) : null;
                if (crawlingFile != null && crawlingFile.exists()) {
                    post.remove("crawlingFile$file");
                }

                // normalize URL
                DigestURI crawlingStartURL = null;
                if (crawlingFile == null) try {crawlingStartURL = new DigestURI(crawlingStart);} catch (final MalformedURLException e1) {Log.logException(e1);}
                crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);

                // set new properties
                final boolean fullDomain = "domain".equals(post.get("range", "wide")); // special property in simple crawl start
                final boolean subPath    = "subpath".equals(post.get("range", "wide")); // special property in simple crawl start

                // set the crawl filter
                String newcrawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL);
                final String newcrawlingMustNotMatch = post.get("mustnotmatch", CrawlProfile.MATCH_NEVER);
                if (newcrawlingMustMatch.length() < 2) newcrawlingMustMatch = CrawlProfile.MATCH_ALL; // avoid that all urls are filtered out if bad value was submitted
                // special cases:
                if (crawlingStartURL!= null && fullDomain) {
                    if (crawlingStartURL.isFile()) {
                        newcrawlingMustMatch = "file://" + crawlingStartURL.getPath() + ".*";
                    } else if (crawlingStartURL.isSMB()) {
                        newcrawlingMustMatch = "smb://.*" + crawlingStartURL.getHost() + ".*" + crawlingStartURL.getPath() + ".*";
                    } else if (crawlingStartURL.isFTP()) {
                        newcrawlingMustMatch = "ftp://.*" + crawlingStartURL.getHost() + ".*" + crawlingStartURL.getPath() + ".*";
                    } else {
                        newcrawlingMustMatch = ".*" + crawlingStartURL.getHost() + ".*";
                    }
                }
                if (crawlingStart!= null && subPath && (pos = crawlingStart.lastIndexOf('/')) > 0) {
                    newcrawlingMustMatch = crawlingStart.substring(0, pos + 1) + ".*";
                }

                final boolean crawlOrder = post.get("crawlOrder", "off").equals("on");
                env.setConfig("crawlOrder", crawlOrder);

                int newcrawlingdepth = post.getInt("crawlingDepth", 8);
                env.setConfig("crawlingDepth", Integer.toString(newcrawlingdepth));
                if ((crawlOrder) && (newcrawlingdepth > 8)) newcrawlingdepth = 8;

                // recrawl
                final String recrawl = post.get("recrawl", "nodoubles"); // nodoubles, reload, scheduler
                boolean crawlingIfOlderCheck = "on".equals(post.get("crawlingIfOlderCheck", "off"));
                int crawlingIfOlderNumber = post.getInt("crawlingIfOlderNumber", -1);
                String crawlingIfOlderUnit = post.get("crawlingIfOlderUnit","year"); // year, month, day, hour
                int repeat_time = post.getInt("repeat_time", -1);
                final String repeat_unit = post.get("repeat_unit", "seldays"); // selminutes, selhours, seldays

                if ("scheduler".equals(recrawl) && repeat_time > 0) {
                    // set crawlingIfOlder attributes that are appropriate for scheduled crawling
                    crawlingIfOlderCheck = true;
                    crawlingIfOlderNumber = "selminutes".equals(repeat_unit) ? 1 : "selhours".equals(repeat_unit) ? repeat_time / 2 : repeat_time * 12;
                    crawlingIfOlderUnit = "hour";
                } else if ("reload".equals(recrawl)) {
                    repeat_time = -1;
                    crawlingIfOlderCheck = true;
                } else if ("nodoubles".equals(recrawl)) {
                    repeat_time = -1;
                    crawlingIfOlderCheck = false;
                }
                final long crawlingIfOlder = recrawlIfOlderC(crawlingIfOlderCheck, crawlingIfOlderNumber, crawlingIfOlderUnit);
                env.setConfig("crawlingIfOlder", crawlingIfOlder);

                // store this call as api call
                if (repeat_time > 0) {
                    // store as scheduled api call
                    sb.tables.recordAPICall(post, "Crawler_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "crawl start for " + ((crawlingStart == null) ? post.get("crawlingFile", "") : crawlingStart), repeat_time, repeat_unit.substring(3));
                } else {
                    // store just a protocol
                    sb.tables.recordAPICall(post, "Crawler_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "crawl start for " + ((crawlingStart == null) ? post.get("crawlingFile", "") : crawlingStart));
                }

                final boolean crawlingDomMaxCheck = "on".equals(post.get("crawlingDomMaxCheck", "off"));
                final int crawlingDomMaxPages = (crawlingDomMaxCheck) ? post.getInt("crawlingDomMaxPages", -1) : -1;
                env.setConfig("crawlingDomMaxPages", Integer.toString(crawlingDomMaxPages));

                final boolean crawlingQ = "on".equals(post.get("crawlingQ", "off"));
                env.setConfig("crawlingQ", crawlingQ);

                final boolean indexText = "on".equals(post.get("indexText", "on"));
                env.setConfig("indexText", indexText);

                final boolean indexMedia = "on".equals(post.get("indexMedia", "on"));
                env.setConfig("indexMedia", indexMedia);

                boolean storeHTCache = "on".equals(post.get("storeHTCache", "on"));
                if (crawlingStartURL!= null &&(crawlingStartURL.isFile() || crawlingStartURL.isSMB())) storeHTCache = false;
                env.setConfig("storeHTCache", storeHTCache);

                CacheStrategy cachePolicy = CacheStrategy.parse(post.get("cachePolicy", "iffresh"));
                if (cachePolicy == null) cachePolicy = CacheStrategy.IFFRESH;

                final boolean xsstopw = "on".equals(post.get("xsstopw", "off"));
                env.setConfig("xsstopw", xsstopw);

                final boolean xdstopw = "on".equals(post.get("xdstopw", "off"));
                env.setConfig("xdstopw", xdstopw);

                final boolean xpstopw = "on".equals(post.get("xpstopw", "off"));
                env.setConfig("xpstopw", xpstopw);

                final String crawlingMode = post.get("crawlingMode","url");
                if (crawlingStart != null && crawlingStart.startsWith("ftp")) {
                    try {
                        // check if the crawl filter works correctly
                        Pattern.compile(newcrawlingMustMatch);
                        final CrawlProfile profile = new CrawlProfile(
                                crawlingStart,
                                crawlingStartURL,
                                newcrawlingMustMatch,
                                CrawlProfile.MATCH_NEVER,
                                newcrawlingdepth,
                                crawlingIfOlder,
                                crawlingDomMaxPages,
                                crawlingQ,
                                indexText,
                                indexMedia,
                                storeHTCache,
                                crawlOrder,
                                xsstopw,
                                xdstopw,
                                xpstopw,
                                cachePolicy);
                        sb.crawler.putActive(profile.handle().getBytes(), profile);
                        sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
                        final DigestURI url = crawlingStartURL;
                        sb.crawlStacker.enqueueEntriesFTP(sb.peers.mySeed().hash.getBytes(), profile.handle(), url.getHost(), url.getPort(), false);
                    } catch (final PatternSyntaxException e) {
                        prop.put("info", "4"); // crawlfilter does not match url
                        prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch);
                        prop.putHTML("info_error", e.getMessage());
                    } catch (final Exception e) {
                        // mist
                        prop.put("info", "7"); // Error with file
                        prop.putHTML("info_crawlingStart", crawlingStart);
                        prop.putHTML("info_error", e.getMessage());
                        Log.logException(e);
                    }
                    sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
                } else if ("url".equals(crawlingMode)) {

                    // check if pattern matches
                    if ((crawlingStart == null || crawlingStartURL == null) /* || (!(crawlingStart.matches(newcrawlingfilter))) */) {
                        // print error message
                        prop.put("info", "4"); //crawlfilter does not match url
                        prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch);
                        prop.putHTML("info_crawlingStart", crawlingStart);
                    } else try {

                        // check if the crawl filter works correctly
                        Pattern.compile(newcrawlingMustMatch);

                        // stack request
                        // first delete old entry, if exists
                        final DigestURI url = new DigestURI(crawlingStart);
                        final byte[] urlhash = url.hash();
                        indexSegment.urlMetadata().remove(urlhash);
                        sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
                        sb.crawlQueues.errorURL.remove(urlhash);

                        // stack url
                        sb.crawler.removePassive(crawlingStartURL.hash()); // if there is an old entry, delete it
                        final CrawlProfile pe = new CrawlProfile(
                                (crawlingStartURL.getHost() == null) ? crawlingStartURL.toNormalform(true, false) : crawlingStartURL.getHost(),
                                crawlingStartURL,
                                newcrawlingMustMatch,
                                newcrawlingMustNotMatch,
                                newcrawlingdepth,
                                crawlingIfOlder,
                                crawlingDomMaxPages,
                                crawlingQ,
                                indexText, indexMedia,
                                storeHTCache,
                                crawlOrder,
                                xsstopw,
                                xdstopw,
                                xpstopw,
                                cachePolicy);
                        sb.crawler.putActive(pe.handle().getBytes(), pe);
                        final String reasonString = sb.crawlStacker.stackCrawl(new Request(
                                sb.peers.mySeed().hash.getBytes(),
                                url,
                                null,
                                "CRAWLING-ROOT",
                                new Date(),
                                pe.handle(),
                                0,
                                0,
                                0,
                                0
                                ));

                        if (reasonString == null) {
                            // create a bookmark from crawl start url
                            final Set<String> tags=ListManager.string2set(BookmarkHelper.cleanTagsString(post.get("bookmarkFolder","/crawlStart")));
                            tags.add("crawlStart");
                            if ("on".equals(post.get("createBookmark","off"))) {
                            final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(crawlingStart, "admin");
                                if (bookmark != null) {
                                    bookmark.setProperty(BookmarksDB.Bookmark.BOOKMARK_TITLE, post.get("bookmarkTitle", crawlingStart));
                                    bookmark.setOwner("admin");
                                    bookmark.setPublic(false);
                                    bookmark.setTags(tags, true);
                                    sb.bookmarksDB.saveBookmark(bookmark);
                                }
                            }
                            // liftoff!
                            prop.put("info", "8");//start msg
                            prop.putHTML("info_crawlingURL", (post.get("crawlingURL")));

                            // generate a YaCyNews if the global flag was set
                            if (!sb.isRobinsonMode() && crawlOrder) {
                                final Map<String, String> m = new HashMap<String, String>(pe); // must be cloned
                                m.remove("specificDepth");
                                m.remove("indexText");
                                m.remove("indexMedia");
                                m.remove("remoteIndexing");
                                m.remove("xsstopw");
                                m.remove("xpstopw");
                                m.remove("xdstopw");
                                m.remove("storeTXCache");
                                m.remove("storeHTCache");
                                m.remove("generalFilter");
                                m.remove("specificFilter");
                                m.put("intention", post.get("intention", "").replace(',', '/'));
                                sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m);
                            }
                        } else {
                            prop.put("info", "5"); //Crawling failed
                            prop.putHTML("info_crawlingURL", (post.get("crawlingURL")));
                            prop.putHTML("info_reasonString", reasonString);

                            sb.crawlQueues.errorURL.push(
                                new Request(
                                        sb.peers.mySeed().hash.getBytes(),
                                        crawlingStartURL,
                                        null,
                                        "",
                                        new Date(),
                                        pe.handle(),
                                        0,
                                        0,
                                        0,
                                        0),
                                sb.peers.mySeed().hash.getBytes(),
                                new Date(),
                                1,
                                FailCategory.FINAL_LOAD_CONTEXT,
                                reasonString, -1);
                        }
                    } catch (final PatternSyntaxException e) {
                        prop.put("info", "4"); // crawlfilter does not match url
                        prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch);
                        prop.putHTML("info_error", e.getMessage());
                    } catch (final Exception e) {
                        // mist
                        prop.put("info", "6"); // Error with url
                        prop.putHTML("info_crawlingStart", crawlingStart);
                        prop.putHTML("info_error", e.getMessage());
                        Log.logException(e);
                    }

                } else if ("file".equals(crawlingMode)) {
                    if (post.containsKey("crawlingFile")) {
                        final String crawlingFileContent = post.get("crawlingFile$file", "");
                        try {
                            // check if the crawl filter works correctly
                            Pattern.compile(newcrawlingMustMatch);
                            final ContentScraper scraper = new ContentScraper(new DigestURI(crawlingFile));
                            final Writer writer = new TransformerWriter(null, null, scraper, null, false);
                            if (crawlingFile != null && crawlingFile.exists()) {
                                FileUtils.copy(new FileInputStream(crawlingFile), writer);
                            } else {
                                FileUtils.copy(crawlingFileContent, writer);
                            }
                            writer.close();

                            // get links and generate filter
                            final Map<MultiProtocolURI, Properties> hyperlinks = scraper.getAnchors();
                            if (fullDomain && newcrawlingdepth > 0) newcrawlingMustMatch = siteFilter(hyperlinks.keySet());

                            final DigestURI crawlURL = new DigestURI("file://" + crawlingFile.toString());
                            final CrawlProfile profile = new CrawlProfile(
                                    crawlingFileName,
                                    crawlURL,
                                    newcrawlingMustMatch,
                                    CrawlProfile.MATCH_NEVER,
                                    newcrawlingdepth,
                                    crawlingIfOlder,
                                    crawlingDomMaxPages,
                                    crawlingQ,
                                    indexText,
                                    indexMedia,
                                    storeHTCache,
                                    crawlOrder,
                                    xsstopw,
                                    xdstopw,
                                    xpstopw,
                                    cachePolicy);
                            sb.crawler.putActive(profile.handle().getBytes(), profile);
                            sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
                            sb.crawlStacker.enqueueEntriesAsynchronous(sb.peers.mySeed().hash.getBytes(), profile.handle(), hyperlinks, true);
                        } catch (final PatternSyntaxException e) {
                            prop.put("info", "4"); // crawlfilter does not match url
                            prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch);
                            prop.putHTML("info_error", e.getMessage());
                        } catch (final Exception e) {
                            // mist
                            prop.put("info", "7"); // Error with file
                            prop.putHTML("info_crawlingStart", crawlingFileName);
                            prop.putHTML("info_error", e.getMessage());
                            Log.logException(e);
                        }
                        sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
                    }
                } else if ("sitemap".equals(crawlingMode)) {
                    final String sitemapURLStr = post.get("sitemapURL","");
                  try {
                    final DigestURI sitemapURL = new DigestURI(sitemapURLStr);
                    final CrawlProfile pe = new CrawlProfile(
                        sitemapURLStr,
                        sitemapURL,
                        CrawlProfile.MATCH_ALL,
                        CrawlProfile.MATCH_NEVER,
                        0,
                        crawlingIfOlder,
                        crawlingDomMaxPages,
                        true,
                        indexText,
                        indexMedia,
                        storeHTCache,
                        crawlOrder,
                        xsstopw,
                        xdstopw,
                        xpstopw,
                        cachePolicy);
                    sb.crawler.putActive(pe.handle().getBytes(), pe);
                    final SitemapImporter importer = new SitemapImporter(sb, sitemapURL, pe);
                    importer.start();
                  } catch (final Exception e) {
                    // mist
                    prop.put("info", "6");//Error with url
                    prop.putHTML("info_crawlingStart", sitemapURLStr);
                    prop.putHTML("info_error", e.getMessage());
                    Log.logException(e);
                  }
                } else if ("sitelist".equals(crawlingMode)) {
                    try {
                        final DigestURI sitelistURL = new DigestURI(crawlingStart);
                        // download document
                        ContentScraper scraper = null;
                        scraper = sb.loader.parseResource(sitelistURL, CacheStrategy.IFFRESH);
                        // String title = scraper.getTitle();
                        // String description = scraper.getDescription();

                        // get links and generate filter
                        final Map<MultiProtocolURI, Properties> hyperlinks = scraper.getAnchors();
                        if (fullDomain && newcrawlingdepth > 0) newcrawlingMustMatch = siteFilter(hyperlinks.keySet());

                        // put links onto crawl queue
                        final CrawlProfile profile = new CrawlProfile(
                                sitelistURL.getHost(),
                                sitelistURL,
                                newcrawlingMustMatch,
                                CrawlProfile.MATCH_NEVER,
                                newcrawlingdepth,
                                crawlingIfOlder,
                                crawlingDomMaxPages,
                                crawlingQ,
                                indexText,
                                indexMedia,
                                storeHTCache,
                                crawlOrder,
                                xsstopw,
                                xdstopw,
                                xpstopw,
                                cachePolicy);
                        sb.crawler.putActive(profile.handle().getBytes(), profile);
                        sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
                        final Iterator<Map.Entry<MultiProtocolURI, Properties>> linkiterator = hyperlinks.entrySet().iterator();
                        DigestURI nexturl;
                        while (linkiterator.hasNext()) {
                            final Map.Entry<MultiProtocolURI, Properties> e = linkiterator.next();
                            if (e.getKey() == null) continue;
                            nexturl = new DigestURI(e.getKey());
                            // remove the url from the database to be prepared to crawl them again
                            final byte[] urlhash = nexturl.hash();
                            indexSegment.urlMetadata().remove(urlhash);
                            sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
                            sb.crawlQueues.errorURL.remove(urlhash);
                            sb.crawlStacker.enqueueEntry(new Request(
                                    sb.peers.mySeed().hash.getBytes(),
                                    nexturl,
                                    null,
                                    e.getValue().getProperty("name", ""),
                                    new Date(),
                                    profile.handle(),
                                    0,
                                    0,
                                    0,
                                    0
                                    ));
                        }
                    } catch (final Exception e) {
                        // mist
                        prop.put("info", "6");//Error with url
                        prop.putHTML("info_crawlingStart", crawlingStart);
                        prop.putHTML("info_error", e.getMessage());
                        Log.logException(e);
                    }
                }
            }
        }

        if (post != null && post.containsKey("crawlingPerformance")) {
            setPerformance(sb, post);
        }

        // performance settings
        final long LCbusySleep = env.getConfigLong(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_BUSYSLEEP, 1000L);
        final int LCppm = (int) (60000L / Math.max(1,LCbusySleep));
        prop.put("crawlingSpeedMaxChecked", (LCppm >= 30000) ? "1" : "0");
        prop.put("crawlingSpeedCustChecked", ((LCppm > 10) && (LCppm < 30000)) ? "1" : "0");
        prop.put("crawlingSpeedMinChecked", (LCppm <= 10) ? "1" : "0");
        prop.put("customPPMdefault", Integer.toString(LCppm));

        // return rewrite properties
        return prop;
    }
View Full Code Here

public class test {
   
    // http://localhost:8090/test.xml?count=10
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final serverObjects prop = new serverObjects();
        final int count = Math.min(1000, (post == null) ? 0 : post.getInt("count", 0));
       
        for (int i = 0; i < count; i++) {
            prop.put("item_" + i + "_text", Integer.toString(i));
        }
        prop.put("item", count);

        return prop;
    }
View Full Code Here

    private static boolean nothingChanged = false;

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final serverObjects prop = new serverObjects();
        final Switchboard sb = (Switchboard) env;

        // get referer for backlink
        final MultiProtocolURI referer = header.referer();
        prop.put("referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true, true));
        //if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());

        if (post == null) {
            prop.put("info", "1");//no information submitted
            return prop;
        }

        // admin password
        if (post.containsKey("adminaccount")) {
            // read and process data
            final String user   = post.get("adminuser");
            final String pw1    = post.get("adminpw1");
            final String pw2    = post.get("adminpw2");
            // do checks
            if ((user == null) || (pw1 == null) || (pw2 == null)) {
                prop.put("info", "1");//error with submitted information
                return prop;
            }
            if (user.length() == 0) {
                prop.put("info", "2");//username must be given
                return prop;
            }
            if (!(pw1.equals(pw2))) {
                prop.put("info", "3");//pw check failed
                return prop;
            }
            // check passed. set account:
            env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(user + ":" + pw1)));
            env.setConfig("adminAccount", "");
            prop.put("info", "5");//admin account changed
            prop.putHTML("info_user", user);
            return prop;
        }


        // proxy password
        if (post.containsKey("proxyaccount")) {
            /*
             * set new port
             */
            final String port = post.get("port");
            prop.putHTML("info_port", port);
            if (!env.getConfig("port", port).equals(port)) {
                // validation port
                final serverCore theServerCore = (serverCore) env.getThread("10_httpd");
                try {
                    final InetSocketAddress theNewAddress = theServerCore.generateSocketAddress(port);
                    final String hostName = Domains.getHostName(theNewAddress.getAddress());
                    prop.put("info_restart", "1");
                    prop.put("info_restart_ip",(hostName.equals("0.0.0.0"))? "localhost" : hostName);
                    prop.put("info_restart_port", theNewAddress.getPort());

                    env.setConfig("port", port);

                    theServerCore.reconnect(5000);
                } catch (final SocketException e) {
                    prop.put("info", "26");
                    return prop;
                }
            } else {
                prop.put("info_restart", "0");
            }

            // read and process data
            String filter = (post.get("proxyfilter")).trim();
            final boolean useProxyAccounts = post.containsKey("use_proxyaccounts") && post.get("use_proxyaccounts").equals("on");
            // do checks
            if (filter == null) {
                prop.put("info", "1");//error with submitted information
                return prop;
            }
            /*if (user.length() == 0) {
                prop.put("info", 2);//username must be given
                return prop;
            }*/
            /*if (!(pw1.equals(pw2))) {
                prop.put("info", 3);//pw check failed
                return prop;
            }*/

            if (filter.length() == 0) filter = "*";
            else if (!filter.equals("*")){
                // testing proxy filter
                int patternCount = 0;
                String patternStr = null;
                try {
                    final StringTokenizer st = new StringTokenizer(filter,",");
                    while (st.hasMoreTokens()) {
                        patternCount++;
                        patternStr = st.nextToken();
                        Pattern.compile(patternStr);
                    }
                } catch (final PatternSyntaxException e) {
                    prop.put("info", "27");
                    prop.putHTML("info_filter", filter);
                    prop.put("info_nr", patternCount);
                    prop.putHTML("info_error", e.getMessage());
                    prop.putHTML("info_pattern", patternStr);
                    return prop;
                }
            }

            // check passed. set account:
            env.setConfig("proxyClient", filter);
            env.setConfig("use_proxyAccounts", useProxyAccounts);
            if (!useProxyAccounts){
                prop.put("info", "6");//proxy account has changed(no pw)
                prop.putHTML("info_filter", filter);
      } else {
                prop.put("info", "7");//proxy account has changed
                //prop.put("info_user", user);
                prop.putHTML("info_filter", filter);
      }
            return prop;
        }

        // http networking
        if (post.containsKey("httpNetworking")) {

            // set transparent proxy flag
            HTTPDProxyHandler.isTransparentProxy = post.containsKey("isTransparentProxy");
            env.setConfig("isTransparentProxy", HTTPDProxyHandler.isTransparentProxy);
            prop.put("info_isTransparentProxy", HTTPDProxyHandler.isTransparentProxy ? "on" : "off");

            // setting the keep alive property
            HTTPDemon.keepAliveSupport = post.containsKey("connectionKeepAliveSupport");
            env.setConfig("connectionKeepAliveSupport", HTTPDemon.keepAliveSupport);
            prop.put("info_connectionKeepAliveSupport", HTTPDemon.keepAliveSupport ? "on" : "off");

            // setting via header property
            env.setConfig("proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader"));
            prop.put("info_proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader")? "on" : "off");

            // setting X-Forwarded-for header property
            env.setConfig("proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader"));
            prop.put("info_proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader")? "on" : "off");

            prop.put("info", "20");
            return prop;
        }

        // server access
        if (post.containsKey("serveraccount")) {

            // static IP
            String staticIP =  (post.get("staticIP")).trim();
            if (staticIP.startsWith("http://")) {
                if (staticIP.length() > 7) { staticIP = staticIP.substring(7); } else { staticIP = ""; }
            } else if (staticIP.startsWith("https://")) {
                if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; }
            }
            // TODO IPv6 support!
            if (staticIP.indexOf(":") > 0) {
                staticIP = staticIP.substring(0, staticIP.indexOf(":"));
            }
            if (staticIP.length() == 0) {
                serverCore.useStaticIP = false;
            } else {
                serverCore.useStaticIP = true;
            }
            if (yacySeed.isProperIP(staticIP) == null) sb.peers.mySeed().setIP(staticIP);
            env.setConfig("staticIP", staticIP);

            // server access data
            String filter = (post.get("serverfilter")).trim();
            /*String user   = (String) post.get("serveruser");
            String pw1    = (String) post.get("serverpw1");
            String pw2    = (String) post.get("serverpw2");*/
            // do checks
            if (filter == null) {
                //if ((filter == null) || (user == null) || (pw1 == null) || (pw2 == null)) {
                prop.put("info", "1");//error with submitted information
                return prop;
            }
           /* if (user.length() == 0) {
                prop.put("info", 2);//username must be given
                return prop;
            }
            if (!(pw1.equals(pw2))) {
                prop.put("info", 3);//pw check failed
                return prop;
            }*/
            if (filter.length() == 0) filter = "*";
            else if (!filter.equals("*")){
                // testing proxy filter
                int patternCount = 0;
                String patternStr = null;
                try {
                    final StringTokenizer st = new StringTokenizer(filter,",");
                    while (st.hasMoreTokens()) {
                        patternCount++;
                        patternStr = st.nextToken();
                        Pattern.compile(patternStr);
                    }
                } catch (final PatternSyntaxException e) {
                    prop.put("info", "27");
                    prop.putHTML("info_filter", filter);
                    prop.put("info_nr", patternCount);
                    prop.putHTML("info_error", e.getMessage());
                    prop.putHTML("info_pattern", patternStr);
                    return prop;
                }
            }

            // check passed. set account:
            env.setConfig("serverClient", filter);

            prop.put("info", "8");//server access filter updated
            //prop.put("info_user", user);
            prop.putHTML("info_filter", filter);
            return prop;
        }

        if (post.containsKey("proxysettings")) {

            /* ====================================================================
             * Reading out the remote proxy settings
             * ==================================================================== */
            final boolean useRemoteProxy = post.containsKey("remoteProxyUse");
            final boolean useRemoteProxy4Yacy = post.containsKey("remoteProxyUse4Yacy");
            final boolean useRemoteProxy4SSL = post.containsKey("remoteProxyUse4SSL");

            final String remoteProxyHost = post.get("remoteProxyHost", "");
            final int remoteProxyPort = post.getInt("remoteProxyPort", 3128);

            final String remoteProxyUser = post.get("remoteProxyUser", "");
            final String remoteProxyPwd = post.get("remoteProxyPwd", "");

            final String remoteProxyNoProxyStr = post.get("remoteProxyNoProxy", "");
            //String[] remoteProxyNoProxyPatterns = remoteProxyNoProxyStr.split(",");

            /* ====================================================================
             * Storing settings into config file
             * ==================================================================== */
            env.setConfig("remoteProxyHost", remoteProxyHost);
            env.setConfig("remoteProxyPort", Integer.toString(remoteProxyPort));
            env.setConfig("remoteProxyUser", remoteProxyUser);
            env.setConfig("remoteProxyPwd", remoteProxyPwd);
            env.setConfig("remoteProxyNoProxy", remoteProxyNoProxyStr);
            env.setConfig("remoteProxyUse", useRemoteProxy);
            env.setConfig("remoteProxyUse4Yacy", useRemoteProxy4Yacy);
            env.setConfig("remoteProxyUse4SSL", useRemoteProxy4SSL);

            /* ====================================================================
             * Enabling settings
             * ==================================================================== */
            sb.initRemoteProxy();

            prop.put("info", "15"); // The remote-proxy setting has been changed
            return prop;
        }

        if (post.containsKey("seedUploadRetry")) {
            String error;
            if ((error = yacyCore.saveSeedList(sb)) == null) {
                // trying to upload the seed-list file
                prop.put("info", "13");
                prop.put("info_success", "1");
            } else {
                prop.put("info", "14");
                prop.putHTML("info_errormsg",error.replaceAll("\n","<br>"));
                env.setConfig("seedUploadMethod","none");
            }
            return prop;
        }

        if (post.containsKey("seedSettings")) {
            // get the currently used uploading method
            final String oldSeedUploadMethod = env.getConfig("seedUploadMethod","none");
            final String newSeedUploadMethod = post.get("seedUploadMethod");
            final String oldSeedURLStr = sb.peers.mySeed().get(yacySeed.SEEDLISTURL, "");
            final String newSeedURLStr = post.get("seedURL");

            final boolean seedUrlChanged = !oldSeedURLStr.equals(newSeedURLStr);
            boolean uploadMethodChanged = !oldSeedUploadMethod.equals(newSeedUploadMethod);
            if (uploadMethodChanged) {
                uploadMethodChanged = yacyCore.changeSeedUploadMethod(newSeedUploadMethod);
            }

            if (seedUrlChanged || uploadMethodChanged) {
                env.setConfig("seedUploadMethod", newSeedUploadMethod);
                sb.peers.mySeed().put(yacySeed.SEEDLISTURL, newSeedURLStr);

                // try an upload
                String error;
                if ((error = yacyCore.saveSeedList(sb)) == null) {
                    // we have successfully uploaded the seed-list file
                    prop.put("info_seedUploadMethod", newSeedUploadMethod);
                    prop.putHTML("info_seedURL",newSeedURLStr);
                    prop.put("info_success", newSeedUploadMethod.equalsIgnoreCase("none") ? "0" : "1");
                    prop.put("info", "19");
                } else {
                    prop.put("info", "14");
                    prop.putHTML("info_errormsg", error.replaceAll("\n","<br>"));
                    env.setConfig("seedUploadMethod","none");
                }
                return prop;
            }
        }

        /*
         * Loop through the available seed uploaders to see if the
         * configuration of one of them has changed
         */
        final HashMap<String, String> uploaders = yacyCore.getSeedUploadMethods();
        final Iterator<String> uploaderKeys = uploaders.keySet().iterator();
        while (uploaderKeys.hasNext()) {
            // get the uploader module name
            final String uploaderName = uploaderKeys.next();


            // determining if the user has reconfigured the settings of this uploader
            if (post.containsKey("seed" + uploaderName + "Settings")) {
                nothingChanged = true;
                final yacySeedUploader theUploader = yacyCore.getSeedUploader(uploaderName);
                final String[] configOptions = theUploader.getConfigurationOptions();
                if (configOptions != null) {
                    for (final String configOption : configOptions) {
                        final String newSettings = post.get(configOption,"");
                        final String oldSettings = env.getConfig(configOption,"");
                        // bitwise AND with boolean is same as logic AND
                        nothingChanged &= newSettings.equals(oldSettings);
                        if (!nothingChanged) {
                            env.setConfig(configOption,newSettings);
                        }
                    }
                }
                if (!nothingChanged) {
                    // if the seed upload method is equal to the seed uploader whose settings
                    // were changed, we now try to upload the seed list with the new settings
                    if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) {
                        String error;
                        if ((error = yacyCore.saveSeedList(sb)) == null) {

                            // we have successfully uploaded the seed file
                            prop.put("info", "13");
                            prop.put("info_success", "1");
                        } else {
                            // if uploading failed we print out an error message
                            prop.put("info", "14");
                            prop.putHTML("info_errormsg",error.replaceAll("\n","<br>"));
                            env.setConfig("seedUploadMethod","none");
                        }
                    } else {
                        prop.put("info", "13");
                        prop.put("info_success", "0");
                    }
                } else {
                    prop.put("info", "13");
                    prop.put("info_success", "0");
                }
                return prop;
            }
        }

        /*
         * Message forwarding configuration
         */
        if (post.containsKey("msgForwarding")) {
            env.setConfig("msgForwardingEnabled", post.containsKey("msgForwardingEnabled"));
            env.setConfig("msgForwardingCmd", post.get("msgForwardingCmd"));
            env.setConfig("msgForwardingTo", post.get("msgForwardingTo"));

            prop.put("info", "21");
            prop.put("info_msgForwardingEnabled", post.containsKey("msgForwardingEnabled") ? "on" : "off");
            prop.putHTML("info_msgForwardingCmd", post.get("msgForwardingCmd"));
            prop.putHTML("info_msgForwardingTo", post.get("msgForwardingTo"));

            return prop;
        }

        // Crawler settings
        if (post.containsKey("crawlerSettings")) {

            // get Crawler Timeout
            String timeoutStr = post.get("crawler.clientTimeout");
            if (timeoutStr==null||timeoutStr.length()==0) timeoutStr = "10000";

            int crawlerTimeout;
            try {
                crawlerTimeout = Integer.parseInt(timeoutStr);
                if (crawlerTimeout < 0) crawlerTimeout = 0;
                env.setConfig("crawler.clientTimeout", Integer.toString(crawlerTimeout));
            } catch (final NumberFormatException e) {
                prop.put("info", "29");
                prop.putHTML("info_crawler.clientTimeout",post.get("crawler.clientTimeout"));
                return prop;
            }

            // get maximum http file size
            String maxSizeStr = post.get("crawler.http.maxFileSize");
            if (maxSizeStr==null||maxSizeStr.length()==0) maxSizeStr = "-1";

            long maxHttpSize;
            try {
                maxHttpSize = Integer.parseInt(maxSizeStr);
                if(maxHttpSize < 0) {
                    maxHttpSize = -1;
                }
                env.setConfig("crawler.http.maxFileSize", Long.toString(maxHttpSize));
            } catch (final NumberFormatException e) {
                prop.put("info", "30");
                prop.putHTML("info_crawler.http.maxFileSize",post.get("crawler.http.maxFileSize"));
                return prop;
            }

            // get maximum ftp file size
            maxSizeStr = post.get("crawler.ftp.maxFileSize");
            if (maxSizeStr==null||maxSizeStr.length()==0) maxSizeStr = "-1";

            long maxFtpSize;
            try {
                maxFtpSize = Integer.parseInt(maxSizeStr);
                env.setConfig("crawler.ftp.maxFileSize", Long.toString(maxFtpSize));
            } catch (final NumberFormatException e) {
                prop.put("info", "31");
                prop.putHTML("info_crawler.ftp.maxFileSize",post.get("crawler.ftp.maxFileSize"));
                return prop;
            }

            maxSizeStr = post.get("crawler.smb.maxFileSize");
            if (maxSizeStr==null||maxSizeStr.length()==0) maxSizeStr = "-1";

            long maxSmbSize;
            try {
                maxSmbSize = Integer.parseInt(maxSizeStr);
                env.setConfig("crawler.smb.maxFileSize", Long.toString(maxSmbSize));
            } catch (final NumberFormatException e) {
                prop.put("info", "31");
                prop.putHTML("info_crawler.smb.maxFileSize",post.get("crawler.smb.maxFileSize"));
                return prop;
            }

            maxSizeStr = post.get("crawler.file.maxFileSize");
            if (maxSizeStr==null||maxSizeStr.length()==0) maxSizeStr = "-1";

            long maxFileSize;
            try {
                maxFileSize = Integer.parseInt(maxSizeStr);
                env.setConfig("crawler.file.maxFileSize", Long.toString(maxFileSize));
            } catch (final NumberFormatException e) {
                prop.put("info", "31");
                prop.putHTML("info_crawler.file.maxFileSize",post.get("crawler.file.maxFileSize"));
                return prop;
            }

            // everything is ok
            prop.put("info_crawler.clientTimeout",(crawlerTimeout==0) ? "0" :yacyPeerActions.formatInterval(crawlerTimeout));
            prop.put("info_crawler.http.maxFileSize",(maxHttpSize==-1)? "-1":Formatter.bytesToString(maxHttpSize));
            prop.put("info_crawler.ftp.maxFileSize", (maxFtpSize==-1) ? "-1":Formatter.bytesToString(maxFtpSize));
            prop.put("info_crawler.smb.maxFileSize", (maxSmbSize==-1) ? "-1":Formatter.bytesToString(maxSmbSize));
            prop.put("info_crawler.file.maxFileSize", (maxFileSize==-1) ? "-1":Formatter.bytesToString(maxFileSize));
            prop.put("info", "28");
            return prop;
        }


        // nothing made
        prop.put("info", "1");//no information submitted
        return prop;
    }
View Full Code Here


public class yacysearchlatestinfo {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final serverObjects prop = new serverObjects();
        //Switchboard sb = (Switchboard) env;

        // find search event
        final String eventID = post.get("eventID", "");
        final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
        if (theSearch == null) {
            // the event does not exist, show empty page
            return prop;
        }
        final QueryParams theQuery = theSearch.getQuery();

        // dynamically update count values
        final int totalcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
        final int offset = theQuery.neededResults() - theQuery.displayResults() + 1;
        prop.put("offset", offset);
        prop.put("itemscount", -1);
        prop.put("itemsperpage", theSearch.getQuery().itemsPerPage);
        prop.put("totalcount", Formatter.number(totalcount, true));
        prop.put("localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true));
        prop.put("localMissCount", Formatter.number(theSearch.getRankingResult().getMissCount(), true));
        prop.put("remoteResourceSize", Formatter.number(theSearch.getRankingResult().getRemoteResourceSize(), true));
        prop.put("remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
        prop.put("remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));

        return prop;
    }
View Full Code Here

TOP

Related Classes of de.anomic.server.serverObjects

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.