Package net.yacy.search

Examples of net.yacy.search.Switchboard$indexingQueueEntry


public class mediawiki_p {

    //http://localhost:8090/mediawiki_p.html?dump=wikipedia.de.xml&title=Kartoffel
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws IOException {
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        prop.put("title", "");
        prop.put("page", "");

        if (post == null) {
            return post;
        }

        final String dump = post.get("dump", null);
        final String title = post.get("title", null);
        if (dump == null || title == null) return post;


        final File dumpFile = new File(sb.getDataPath(), "DATA/HTCACHE/mediawiki/" + dump);
        if (!dumpFile.exists()) return post;
        MediawikiImporter.checkIndex(dumpFile);
        final MediawikiImporter.wikisourcerecord w = MediawikiImporter.find(title.replaceAll(" ", "_"), MediawikiImporter.idxFromMediawikiXML(dumpFile));
        if (w == null) {
            return post;
View Full Code Here


public class BlacklistImpExp_p {
    private final static String DISABLED = "disabled_";

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
       
        // initialize the list manager
        ListManager.switchboard = (Switchboard) env;
        ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
       
View Full Code Here

        }
    }

    public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

        final Switchboard sb = (Switchboard)env;

        // the url to the image can be either submitted with an url in clear text, or using a license key
        // if the url is given as clear text, the user must be authorized as admin
        // the license can be used also from non-authorized users

        String urlString = post.get("url", "");
        final String urlLicense = post.get("code", "");
        final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header); // handle access rights

        DigestURI url = null;
        if ((urlString.length() > 0) && (auth)) try {
            url = new DigestURI(urlString);
        } catch (final MalformedURLException e1) {
            url = null;
        }

        if ((url == null) && (urlLicense.length() > 0)) {
            url = sb.licensedURLs.releaseLicense(urlLicense);
            urlString = (url == null) ? null : url.toNormalform(true, true);
        }

        if (urlString == null) return null;

        int width = post.getInt("width", 0);
        int height = post.getInt("height", 0);
        int maxwidth = post.getInt("maxwidth", 0);
        int maxheight = post.getInt("maxheight", 0);

        // get the image as stream
        if (MemoryControl.shortStatus()) iconcache.clear();
        Image image = iconcache.get(urlString);
        if (image == null) {
            byte[] resourceb = null;
            if (url != null) try {
                resourceb = sb.loader.loadContent(sb.loader.request(url, false, true), CacheStrategy.IFEXIST);
            } catch (final IOException e) {
                Log.logFine("ViewImage", "cannot load: " + e.getMessage());
            }
            byte[] imgb = null;
            if (resourceb == null) {
                if (urlString.endsWith(".ico")) {
                    // load default favicon dfltfvcn.ico
                    if (defaulticonb == null) try {
                        imgb = FileUtils.read(new File(sb.getAppPath(), defaulticon));
                    } catch (final IOException e) {
                        return null;
                    } else {
                        imgb = defaulticonb;
                    }
View Full Code Here

public class AccessPicture_p {

    private static int[] times = new int[]{60000, 50000, 40000, 30000, 20000, 10000, 1000};

    public static RasterPlotter respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;

        String color_text    = "AAAAAA";
        String color_back    = "FFFFFF";
        String color_grid    = "333333";
        String color_dot     = "33CC33";
        String color_line    = "555555";

        int width = 1024;
        int height = 576;
        int cellsize = 18;
        boolean corona = false;
        int coronaangle = 0;

        if (post != null) {
            width         = post.getInt("width", 1024);
            height        = post.getInt("height", 576);
            cellsize      = post.getInt("cellsize", cellsize);
            color_text    = post.get("colortext",   color_text);
            color_back    = post.get("colorback",   color_back);
            color_grid    = post.get("colorgrid",   color_grid);
            color_dot     = post.get("colordot",    color_dot);
            color_line    = post.get("colorline",   color_line);
            corona        = post.getBoolean("corona", true);
            coronaangle   = (corona) ? post.getInt("coronaangle", 0) : -1;
        }
        if (coronaangle < 0) corona = false;

        // too small values lead to an error, too big to huge CPU/memory consumption, resulting in possible DOS.
        if (width < 32 ) width = 32;
        if (width > 10000) width = 10000;
        if (height < 24) height = 24;
        if (height > 10000) height = 10000;

        final RasterPlotter.DrawMode drawMode = (RasterPlotter.darkColor(color_back)) ? RasterPlotter.DrawMode.MODE_ADD : RasterPlotter.DrawMode.MODE_SUB;
        final HexGridPlotter picture = new HexGridPlotter(width, height, drawMode, color_back, cellsize);
        picture.drawGrid(color_grid);

        // calculate dimensions for left and right column
        final int gridLeft = 0;
        int gridRight = picture.gridWidth() - 2;
        if ((gridRight & 1) == 0) gridRight--;

        // draw home peer
        final int centerx = (picture.gridWidth() >> 1) - 1;
        final int centery = picture.gridHeight() >> 1;
        picture.setColor(color_dot);
        picture.gridDot(centerx, centery, 5, true, 100);
        if (corona) {
            for (int i = 0; i < 6; i++) {
                picture.gridDot(centerx, centery, 50, i * 60 + coronaangle / 6, i * 60 + 30 + coronaangle / 6);
            }
        } else {
            picture.gridDot(centerx, centery, 50, false, 100);
        }
        //picture.gridDot(centerx, centery, 31, false);
        picture.setColor(color_text);
        picture.gridPrint(centerx, centery, 5, "THIS YACY PEER", "\"" + sb.peers.myName().toUpperCase() + "\"", 0);

        // left column: collect data for access from outside
        final int verticalSlots = (picture.gridHeight() >> 1) - 1;
        final String[] hosts = new String[verticalSlots];
        final int[] time = new int[verticalSlots];
        final int[] count = new int[verticalSlots];
        for (int i = 0; i < verticalSlots; i++) {hosts[i] = null; time[i] = 0; count[i] = 0;}

        String host;
        int c, h;
        for (final int time2 : times) {
            final Iterator<String> i = sb.accessHosts();
            try {
                while (i.hasNext()) {
                    host = i.next();
                    c = sb.latestAccessCount(host, time2);
                    if (c > 0) {
                        h = (Math.abs(host.hashCode())) % hosts.length;
                        hosts[h] = host;
                        count[h] = c;
                        time[h] = time2;
View Full Code Here

public class CrawlStartExpert_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();

        // define visible variables
        prop.put("starturl", /*(intranet) ? repository :*/ "http://");
        prop.put("proxyPrefetchDepth", env.getConfig("proxyPrefetchDepth", "0"));
        prop.put("crawlingDepth", Math.min(3, env.getConfigLong("crawlingDepth", 0)));
        prop.put("directDocByURLChecked", sb.getConfigBool("crawlingDirectDocByURL", true) ? "1" : "0");
        prop.put("mustmatch", /*(intranet) ? repository + ".*" :*/ CrawlProfile.MATCH_ALL_STRING);
        prop.put("mustnotmatch", CrawlProfile.MATCH_NEVER_STRING);
        prop.put("ipMustmatch", sb.getConfig("crawlingIPMustMatch", CrawlProfile.MATCH_ALL_STRING));
        prop.put("ipMustnotmatch", sb.getConfig("crawlingIPMustNotMatch", CrawlProfile.MATCH_NEVER_STRING));
        prop.put("countryMustMatch", sb.getConfig("crawlingCountryMustMatch", ""));

        prop.put("crawlingIfOlderCheck", "0");
        prop.put("crawlingIfOlderUnitYearCheck", "0");
        prop.put("crawlingIfOlderUnitMonthCheck", "0");
        prop.put("crawlingIfOlderUnitDayCheck", "1");
View Full Code Here

              final File newconfFile = new File(dataHome, newconf);
                if(!oldconffile.renameTo(newconfFile))
                    Log.logSevere("STARTUP", "WARNING: the file " + oldconffile + " can not be renamed to "+ newconfFile +"!");
            }
            try {
                sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), newconf);
            } catch (final RuntimeException e) {
                Log.logSevere("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
                System.exit(-1);
            }
            //sbSync.V(); // signal that the sb reference was set
View Full Code Here

public class Network {

    private static final String STR_TABLE_LIST = "table_list_";

    public static serverObjects respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch switchboard) {
        final Switchboard sb = (Switchboard) switchboard;
        final long start = System.currentTimeMillis();

        final serverObjects prop = new serverObjects();
        prop.put("menu", post == null ? 2 : (post.get("menu", "").equals("embed")) ? 0 : (post.get("menu","").equals("simple")) ? 1 : 2);
        if (sb.peers.mySeed() != null) prop.put("menu_newpeer_peerhash", sb.peers.mySeed().hash);

        prop.setLocalized(!(requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"));
        prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
        prop.putHTML("page_networkName", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"));
        final boolean overview = (post == null) || (post.get("page", "0").equals("0"));

        final String mySeedType = sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN);
        final boolean iAmActive = (mySeedType.equals(Seed.PEERTYPE_SENIOR) || mySeedType.equals(Seed.PEERTYPE_PRINCIPAL));

        if (overview) {
            long accActLinks = sb.peers.countActiveURL();
            long accActWords = sb.peers.countActiveRWI();
            final long accPassLinks = sb.peers.countPassiveURL();
            final long accPassWords = sb.peers.countPassiveRWI();
            long accPotLinks = sb.peers.countPotentialURL();
            long accPotWords = sb.peers.countPotentialRWI();

            int conCount = sb.peers.sizeConnected();
            final int disconCount = sb.peers.sizeDisconnected();
            int potCount = sb.peers.sizePotential();

//          final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
            final long otherppm = sb.peers.countActivePPM();
            final double otherqpm = sb.peers.countActiveQPM();
            long myppm = 0;
            double myqph = 0d;

            // create own peer info
            final Seed seed = sb.peers.mySeed();
            if (sb.peers.mySeed() != null){ //our Peer
                // update seed info
                sb.updateMySeed();

                final long LCount = seed.getLinkCount();
                final long ICount = seed.getWordCount();
                final long RCount = seed.getLong(Seed.RCOUNT, 0L);

                // my-info
                prop.putHTML("table_my-name", seed.get(Seed.NAME, "-") );
                prop.put("table_my-hash", seed.hash );
                if (sb.peers.mySeed().isVirgin()) {
                    prop.put("table_my-info", 0);
                } else if(sb.peers.mySeed().isJunior()) {
                    prop.put("table_my-info", 1);
                    accPotLinks += LCount;
                    accPotWords += ICount;
                } else if(sb.peers.mySeed().isSenior()) {
                    prop.put("table_my-info", 2);
                    accActLinks += LCount;
                    accActWords += ICount;
                } else if(sb.peers.mySeed().isPrincipal()) {
                    prop.put("table_my-info", 3);
                    accActLinks += LCount;
                    accActWords += ICount;
                }
                prop.put("table_my-acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0);
                prop.put("table_my-dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0);


                myppm = sb.currentPPM();
                myqph = 60d * sb.averageQPM();
                prop.put("table_my-version", seed.get(Seed.VERSION, "-"));
                prop.put("table_my-utc", seed.get(Seed.UTC, "-"));
                prop.put("table_my-uptime", PeerActions.formatInterval(60000 * seed.getLong(Seed.UPTIME, 0)));
                prop.putNum("table_my-LCount", LCount);
                prop.putNum("table_my-ICount", ICount);
                prop.putNum("table_my-RCount", RCount);
                prop.putNum("table_my-sI", seed.getLong(Seed.INDEX_OUT, 0L));
                prop.putNum("table_my-sU", seed.getLong(Seed.URL_OUT, 0L));
                prop.putNum("table_my-rI", seed.getLong(Seed.INDEX_IN, 0L));
                prop.putNum("table_my-rU", seed.getLong(Seed.URL_IN, 0L));
                prop.putNum("table_my-ppm", myppm);
                prop.putNum("table_my-qph", Math.round(100d * myqph) / 100d);
                prop.putNum("table_my-qph-publocal", Math.round(6000d * sb.averageQPMPublicLocal()) / 100d);
                prop.putNum("table_my-qph-pubremote", Math.round(6000d * sb.averageQPMGlobal()) / 100d);
                prop.putNum("table_my-seeds", seed.getLong(Seed.SCOUNT, 0L));
                prop.putNum("table_my-connects", seed.getFloat(Seed.CCOUNT, 0F));
                prop.put("table_my-url", seed.get(Seed.SEEDLISTURL, ""));

                // generating the location string
                prop.putHTML("table_my-location", ClientIdentification.generateLocation());
            }

            // overall results: Network statistics
            if (iAmActive) conCount++; else if (mySeedType.equals(Seed.PEERTYPE_JUNIOR)) potCount++;
            final int activeLastMonth = sb.peers.sizeActiveSince(30 * 1440);
            final int activeLastWeek = sb.peers.sizeActiveSince(7 * 1440);
            final int activeLastDay = sb.peers.sizeActiveSince(1440);
            final int activeSwitch =
                (activeLastDay <= conCount) ? 0 :
                (activeLastWeek <= activeLastDay) ? 1 :
                (activeLastMonth <= activeLastWeek) ? 2 : 3;
            prop.putNum("table_active-switch", activeSwitch);
            prop.putNum("table_active-switch_last-month", activeLastMonth);
            prop.putNum("table_active-switch_last-week", activeLastWeek);
            prop.putNum("table_active-switch_last-day", activeLastDay);
            prop.putNum("table_active-count", conCount);
            prop.putNum("table_active-links", accActLinks);
            prop.putNum("table_active-words", accActWords);
            prop.putNum("table_passive-count", disconCount);
            prop.putNum("table_passive-links", accPassLinks);
            prop.putNum("table_passive-words", accPassWords);
            prop.putNum("table_potential-count", potCount);
            prop.putNum("table_potential-links", accPotLinks);
            prop.putNum("table_potential-words", accPotWords);
            prop.putNum("table_all-count", conCount + disconCount + potCount);
            prop.putNum("table_all-links", accActLinks + accPassLinks + accPotLinks);
            prop.putNum("table_all-words", accActWords + accPassWords + accPotWords);

            prop.putNum("table_gppm", otherppm + ((iAmActive) ? myppm : 0));
            prop.putNum("table_gqph", Math.round(6000d * otherqpm + 100d * ((iAmActive) ? myqph : 0d)) / 100d);
            prop.put("table", 2); // triggers overview
            prop.put("page", 0);
        } else if (post != null && post.getInt("page", 1) == 4) {
            prop.put("table", 4); // triggers overview
            prop.put("page", 4);

            if (sb.peers.mySeed() != null) {
              prop.put("table_my-hash", sb.peers.mySeed().hash );
              prop.put("table_my-ip", sb.peers.mySeed().getIP() );
              prop.put("table_my-port", sb.peers.mySeed().getPort() );
            }

            if (post.containsKey("addPeer")) {

                // AUTHENTICATE
                if (!requestHeader.containsKey(RequestHeader.AUTHORIZATION)) {
                    prop.putHTML("AUTHENTICATE","log-in");
                    return prop;
                }

                final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();
                map.put(Seed.IP, post.get("peerIP"));
                map.put(Seed.PORT, post.get("peerPort"));
                Seed peer = new Seed(post.get("peerHash"), map);

                sb.updateMySeed();
                final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName());

                if (added <= 0) {
                    prop.put("table_comment",1);
                    prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
                } else {
                    peer = sb.peers.getConnected(peer.hash);
                    if (peer == null) {
                        prop.put("table_comment",1);
                        prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
                    } else {
                        prop.put("table_comment",2);
                        prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress());
                        prop.putHTML("table_comment_details",peer.toString());
                    }
                }

                prop.putHTML("table_peerHash",post.get("peerHash"));
                prop.putHTML("table_peerIP",post.get("peerIP"));
                prop.putHTML("table_peerPort",post.get("peerPort"));
            } else {
                prop.put("table_peerHash","");
                prop.put("table_peerIP","");
                prop.put("table_peerPort","");

                prop.put("table_comment",0);
            }
        } else {
            // generate table
            final int page = (post == null ? 1 : post.getInt("page", 1));
            final int maxCount = (post == null ? 9000 : post.getInt("maxCount", 9000));
            int conCount = 0;
            if (sb.peers == null) {
                prop.put("table", 0);//no remote senior/principal proxies known"
            } else {
                int size = 0;
                switch (page) {
                    case 1 : size = sb.peers.sizeConnected(); break;
                    case 2 : size = sb.peers.sizeDisconnected(); break;
                    case 3 : size = sb.peers.sizePotential(); break;
                    default: break;
                }
                if (size == 0) {
                    prop.put("table", 0);//no remote senior/principal proxies known"
                } else {
                    // add temporary the own seed to the database
                    if (iAmActive) {
                        sb.updateMySeed();
                        sb.peers.addConnected(sb.peers.mySeed());
                    }

                    // find updated Information using YaCyNews
                    final HashSet<String> updatedProfile = new HashSet<String>();
View Full Code Here

import de.anomic.server.serverSwitch;

public class CookieMonitorIncoming_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch sb) {
        final Switchboard switchboard = (Switchboard) sb;

        // return variable that accumulates replacements
        final serverObjects prop = new serverObjects();

  // handle on/off button
        if(post != null) {
            if(post.containsKey("enableCookieMonitoring")) {
                switchboard.setConfig("proxy.monitorCookies", true);
            } else if (post.containsKey("disableCookieMonitoring")) {
                switchboard.setConfig("proxy.monitorCookies", false);
    switchboard.incomingCookies.clear();
    switchboard.outgoingCookies.clear();
            }
        }
        prop.put("monitorCookies.on", switchboard.getConfigBool("proxy.monitorCookies", false) ? "1":"0");
        prop.put("monitorCookies.off", !switchboard.getConfigBool("proxy.monitorCookies", false) ? "1":"0");

        final int maxCount = 100;
        int entCount = 0;
        int tmpCount = 0;
        boolean dark = true;
View Full Code Here

//      if (date == null) return ""; else return dayFormatter.format(date);
//  }

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();

//      int showIndexedCount = 20;
//      boolean se = false;
View Full Code Here

import de.anomic.server.serverSwitch;

public class Tables_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();

        prop.put("showtable", 0);
        prop.put("showedit", 0);
        prop.put("showselection", 0);
View Full Code Here

TOP

Related Classes of net.yacy.search.Switchboard$indexingQueueEntry

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.