Package de.anomic.server

Examples of de.anomic.server.serverObjects

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
        int compareyacy = 0;
        if (post != null && post.getBoolean("compare_yacy", false))
    compareyacy = 1;
       
        final serverObjects prop = new serverObjects();
        prop.put("compareyacy", compareyacy);
        prop.putXML("compareyacy_thisaddress", thisaddress);
        prop.putXML("thisaddress", thisaddress);
        prop.putXML("SearchPageGreeting", promoteSearchPageGreeting);
        prop.putXML("clientname", sb.peers.mySeed().getName());
       
        // return rewrite properties
        return prop;
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
public class IndexCleaner_p {
    private static MetadataRepository.BlacklistCleaner urldbCleanerThread = null;
    private static Segment.ReferenceCleaner indexCleanerThread = null;

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final serverObjects prop = new serverObjects();
        final Switchboard sb = (Switchboard) env;
        prop.put("title", "DbCleanup_p");
       
        // get segment
        Segment indexSegment = null;
        if (post != null && post.containsKey("segment")) {
            String segmentName = post.get("segment");
            if (sb.indexSegments.segmentExist(segmentName)) {
                indexSegment = sb.indexSegments.segment(segmentName);
            }
        } else {
            // take default segment
            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }
       
        if (post!=null) {
            if (post.get("action").equals("ustart")) {
                if (urldbCleanerThread==null || !urldbCleanerThread.isAlive()) {
                    urldbCleanerThread = indexSegment.urlMetadata().getBlacklistCleaner(Switchboard.urlBlacklist, sb.crawlStacker);
                    urldbCleanerThread.start();
                }
                else {
                    urldbCleanerThread.endPause();
                }
            }
            else if (post.get("action").equals("ustop") && (urldbCleanerThread!=null)) {
                urldbCleanerThread.abort();
            }
            else if (post.get("action").equals("upause") && (urldbCleanerThread!=null)) {
                urldbCleanerThread.pause();
            }
            else if (post.get("action").equals("rstart")) {
                if (indexCleanerThread==null || !indexCleanerThread.isAlive()) {
                    indexCleanerThread = indexSegment.getReferenceCleaner(post.get("wordHash","AAAAAAAAAAAA").getBytes());
                    indexCleanerThread.start();
                }
                else {
                    indexCleanerThread.endPause();
                }
            }
            else if (post.get("action").equals("rstop") && (indexCleanerThread!=null)) {
                indexCleanerThread.abort();
            }
            else if (post.get("action").equals("rpause") && (indexCleanerThread!=null)) {
                indexCleanerThread.pause();
            }
            prop.put("LOCATION","");
            return prop;
        }
        if (urldbCleanerThread!=null) {
            prop.put("urldb", "1");
            prop.putNum("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/indexSegment.urlMetadata().size())*100);
            prop.putNum("urldb_blacklisted", urldbCleanerThread.blacklistedUrls);
            prop.putNum("urldb_total", urldbCleanerThread.totalSearchedUrls);
            prop.putHTML("urldb_lastBlacklistedUrl", urldbCleanerThread.lastBlacklistedUrl);
            prop.put("urldb_lastBlacklistedHash", urldbCleanerThread.lastBlacklistedHash);
            prop.putHTML("urldb_lastUrl", urldbCleanerThread.lastUrl);
            prop.put("urldb_lastHash", urldbCleanerThread.lastHash);
            prop.put("urldb_threadAlive", Boolean.toString(urldbCleanerThread.isAlive()));
            prop.put("urldb_threadToString", urldbCleanerThread.toString());
            final double percent = ((double)urldbCleanerThread.blacklistedUrls/urldbCleanerThread.totalSearchedUrls)*100;
            prop.putNum("urldb_percent", percent);
        }
        if (indexCleanerThread!=null) {
            prop.put("rwidb", "1");
            prop.put("rwidb_threadAlive", Boolean.toString(indexCleanerThread.isAlive()));
            prop.put("rwidb_threadToString", indexCleanerThread.toString());
            prop.putNum("rwidb_RWIcountstart", indexCleanerThread.rwiCountAtStart);
            prop.putNum("rwidb_RWIcountnow", indexCleanerThread.rwisize());
            prop.put("rwidb_wordHashNow", (indexCleanerThread.wordHashNow == null) ? "NULL" : ASCII.String(indexCleanerThread.wordHashNow));
            prop.put("rwidb_lastWordHash", (indexCleanerThread.lastWordHash == null) ? "null" : ASCII.String(indexCleanerThread.lastWordHash));
            prop.putNum("rwidb_lastDeletionCounter", indexCleanerThread.lastDeletionCounter);

        }
        return prop;
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    private final static String LANG_FILENAME_FILTER = "^.*\\.lng$";

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

        final serverObjects prop = new serverObjects();
        final String langPath = env.getDataPath("locale.work", "DATA/LOCALE/locales").getAbsolutePath();

        //Fallback
        //prop.put("currentlang", ""); //is done by Translationtemplate
        prop.put("status", "0");//nothing

        List<String> langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER);
        if (langFiles == null) {
            return prop;
        }

        if (post != null) {
            final String selectedLanguage = post.get("language");

            // store this call as api call
            ((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
           
            //change language
            if (post.containsKey("use_button") && selectedLanguage != null){
                /* Only change language if filename is contained in list of filesnames
                 * read from the language directory. This is very important to prevent
                 * directory traversal attacks!
                 */
                if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) {
                    Translator.changeLang(env, langPath, selectedLanguage);
                }

                //delete language file
            } else if (post.containsKey("delete")) {

                /* Only delete file if filename is contained in list of filesnames
                 * read from the language directory. This is very important to prevent
                 * directory traversal attacks!
                 */
                if (langFiles.contains(selectedLanguage)) {
                    final File langfile= new File(langPath, selectedLanguage);
                    FileUtils.deletedelete(langfile);
                }

                //load language file from URL
            } else if (post.containsKey("url")){
                final String url = post.get("url");
                Iterator<String> it;
                try {
                    final DigestURI u = new DigestURI(url);
                    it = FileUtils.strings(u.get(ClientIdentification.getUserAgent(), 10000));
                } catch(final IOException e) {
                    prop.put("status", "1");//unable to get url
                    prop.put("status_url", url);
                    return prop;
                }
                try {
                    final File langFile = new File(langPath, url.substring(url.lastIndexOf('/'), url.length()));
                    final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(langFile)));

                    while (it.hasNext()) {
                        bw.write(it.next() + "\n");
                    }
                    bw.close();
                } catch(final IOException e) {
                    prop.put("status", "2");//error saving the language file
                    return prop;
                }
                if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {
                    Translator.changeLang(env, langPath, url.substring(url.lastIndexOf('/'), url.length()));
                }
            }
        }

        //re-read language files
        langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER);
        Collections.sort(langFiles);
        final Map<String, String> langNames = Translator.langMap(env);

        //virtual entry
        prop.put("langlist_0_file", "default");
        prop.put("langlist_0_name", ((langNames.get("default") == null) ? "default" : langNames.get("default")));
        prop.put("langlist_0_selected", "selected=\"selected\"");

        int count = 0;
        for (final String langFile : langFiles) {
            //+1 because of the virtual entry "default" at top
            final String langKey = langFile.substring(0, langFile.length() -4);
            final String langName = langNames.get(langKey);
            prop.put("langlist_" + (count + 1) + "_file", langFile);
            prop.put("langlist_" + (count + 1) + "_name", ((langName == null) ? langKey : langName));
           
            if(env.getConfig("locale.language", "default").equals(langKey)) {
                prop.put("langlist_" + (count + 1) + "_selected", "selected=\"selected\"");
                prop.put("langlist_0_selected", " "); // reset Default
            } else {
                prop.put("langlist_" + (count + 1) + "_selected", " ");
            }
            count++;
        }
        prop.put("langlist", (count + 1));

        //is done by Translationtemplate
        //langName = (String) langNames.get(env.getConfig("locale.language", "default"));
        //prop.put("currentlang", ((langName == null) ? "default" : langName));
        return prop;
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
public class ConfigProfile_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        final Properties profile = new Properties();
        FileInputStream fileIn = null;
        try {
            fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
            profile.load(fileIn);
        } catch(final IOException e) {
        } finally {
            if (fileIn != null) try { fileIn.close(); } catch (final Exception e) {}
        }

        if (post != null && post.containsKey("set")) {
            profile.setProperty("name", post.get("name"));
            profile.setProperty("nickname", post.get("nickname"));
            profile.setProperty("homepage", post.get("homepage"));
            profile.setProperty("email", post.get("email"));

            profile.setProperty("icq", post.get("icq"));
            profile.setProperty("jabber", post.get("jabber"));
            profile.setProperty("yahoo", post.get("yahoo"));
            profile.setProperty("msn", post.get("msn"));
            profile.setProperty("skype", post.get("skype"));

            profile.setProperty("comment", post.get("comment"));


            prop.putHTML("name", profile.getProperty("name", ""));
            prop.putHTML("nickname", profile.getProperty("nickname", ""));
            prop.putHTML("homepage", profile.getProperty("homepage", ""));
            prop.putHTML("email", profile.getProperty("email", ""));

            prop.putHTML("icq", profile.getProperty("icq", ""));
            prop.putHTML("jabber", profile.getProperty("jabber", ""));
            prop.putHTML("yahoo", profile.getProperty("yahoo", ""));
            prop.putHTML("msn", profile.getProperty("msn", ""));
            prop.putHTML("skype", profile.getProperty("skype", ""));

            prop.putHTML("comment", profile.getProperty("comment", ""));

            // write new values
            FileOutputStream fileOut = null;
            try {
                fileOut = new FileOutputStream(new File("DATA/SETTINGS/profile.txt"));
                profile.store(fileOut , null );

                // generate a news message
                final Properties news = profile;
                news.remove("comment");
                sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news);
                //yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile));
            } catch(final IOException e) {
            } finally {
                if (fileOut != null) try { fileOut.close(); } catch (final Exception e) {}
            }
        }

        else{
            prop.putHTML("name", profile.getProperty("name", ""));
            prop.putHTML("nickname", profile.getProperty("nickname", ""));
            prop.putHTML("homepage", profile.getProperty("homepage", ""));
            prop.putHTML("email", profile.getProperty("email", ""));

            prop.putHTML("icq", profile.getProperty("icq", ""));
            prop.putHTML("jabber", profile.getProperty("jabber", ""));
            prop.putHTML("yahoo", profile.getProperty("yahoo", ""));
            prop.putHTML("msn", profile.getProperty("msn", ""));
            prop.putHTML("skype", profile.getProperty("skype", ""));

            prop.putHTML("comment", profile.getProperty("comment", ""));
        }

        return prop;
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
public class ConfigUpdate_p {

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final serverObjects prop = new serverObjects();
        final Switchboard sb = (Switchboard) env;

        // set if this should be visible
        if (yacyBuildProperties.isPkgManager()) {
            prop.put("candeploy", "2");
            return prop;
        } else if (OS.canExecUnix || OS.isWindows) {
            // we can deploy a new system with (i.e.)
            // cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy
            prop.put("candeploy", "1");
        } else {
            prop.put("candeploy", "0");
        }
       

        prop.put("candeploy_configCommit", "0");
        prop.put("candeploy_autoUpdate", "0");
        prop.put("candeploy_downloadsAvailable", "0");
       
        if (post != null) {
            // check if update is supposed to be installed and a release is defined
            if (post.containsKey("update") && !post.get("releaseinstall", "").isEmpty()) {
                prop.put("forwardToSteering", "1");
                prop.putHTML("forwardToSteering_release",post.get("releaseinstall", ""));
                prop.put("deploys", "1");
                prop.put("candeploy", "2"); // display nothing else
                return prop;
            }
           
            if (post.containsKey("downloadRelease")) {
                // download a release
                final String release = post.get("releasedownload", "");
                if (!release.isEmpty()) {
                    try {
                  yacyRelease versionToDownload = new yacyRelease(new DigestURI(release));
                 
                  // replace this version with version which contains public key
                  yacyRelease.DevAndMainVersions allReleases = yacyRelease.allReleases(false, false);
                  final Set<yacyRelease> mostReleases = versionToDownload.isMainRelease() ? allReleases.main : allReleases.dev;
                  for (final yacyRelease rel : mostReleases) {
                      if (rel.equals(versionToDownload)) {
                    versionToDownload = rel;
                    break;
                      }
                  }
                  versionToDownload.downloadRelease();
                    } catch (final IOException e) {
                  // TODO Auto-generated catch block
                        Log.logException(e);
                    }
                }
            }
        
            if (post.containsKey("checkRelease")) {
                yacyRelease.allReleases(true, false);
            }

            if (post.containsKey("deleteRelease")) {
                final String release = post.get("releaseinstall", "");
                if (!release.isEmpty()) {
                    try {
                        FileUtils.deletedelete(new File(sb.releasePath, release));
                        FileUtils.deletedelete(new File(sb.releasePath, release + ".sig"));
                    } catch (final NullPointerException e) {
                        sb.getLog().logSevere("AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage());
                    }
                }
            }
        
            if (post.containsKey("autoUpdate")) {
                final yacyRelease updateVersion = yacyRelease.rulebasedUpdateInfo(true);
                if (updateVersion == null) {
                    prop.put("candeploy_autoUpdate", "2"); // no more recent release found
                } else {
                    // there is a version that is more recent. Load it and re-start with it
                    sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
                    final File downloaded = updateVersion.downloadRelease();
                    prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.getName());
                    final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
                    if (devenvironment) {
                        sb.getLog().logInfo("AUTO-UPDATE: omitting update because this is a development environment");
                        prop.put("candeploy_autoUpdate", "3");
                    } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
                        sb.getLog().logInfo("AUTO-UPDATE: omitting update because download failed (file cannot be found, is too small or signature was bad)");
                        prop.put("candeploy_autoUpdate", "4");
                    } else {
                        yacyRelease.deployRelease(downloaded);
                        sb.terminate(10, "manual release update to " + downloaded.getName());
                        sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
                        prop.put("candeploy_autoUpdate", "1");
                    }
                }
            }
        
            if (post.containsKey("configSubmit")) {
                prop.put("candeploy_configCommit", "1");
                sb.setConfig("update.process", ("manual".equals(post.get("updateMode", "manual"))) ? "manual" : "auto");
                sb.setConfig("update.cycle", Math.max(12, post.getLong("cycle", 168)));
                sb.setConfig("update.blacklist", post.get("blacklist", ""));
                sb.setConfig("update.concept", ("any".equals(post.get("releaseType", "any"))) ? "any" : "main");
                sb.setConfig("update.onlySignedFiles", (post.getBoolean("onlySignedFiles", false)) ? "1" : "0");
            }
        }
       
        // version information
        final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
        prop.putHTML("candeploy_versionpp", versionstring);
        final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
        float thisVersion = Float.parseFloat(yacyBuildProperties.getVersion());
        // cut off the SVN Rev in the Version
        try {
            thisVersion = (float) (Math.round(thisVersion*1000.0)/1000.0);
        } catch (final NumberFormatException e) {}

           
        // list downloaded releases
        final File[] downloadedFiles = sb.releasePath.listFiles();
        // list can be null if RELEASE directory has been deleted manually
        final int downloadedFilesNum = (downloadedFiles == null) ? 0 : downloadedFiles.length;
           
        prop.put("candeploy_deployenabled", (downloadedFilesNum == 0) ? "0" : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed
         
        final NavigableSet<yacyRelease> downloadedReleases = new TreeSet<yacyRelease>();
        for (final File downloaded : downloadedFiles) {
            try {
                yacyRelease release = new yacyRelease(downloaded);
                downloadedReleases.add(release);
            } catch (final RuntimeException e) {
                // not a valid release
              // can be also a restart- or deploy-file
                final File invalid = downloaded;
                if (!(invalid.getName().endsWith(".bat") || invalid.getName().endsWith(".sh") || invalid.getName().endsWith(".sig"))) { // Windows & Linux don't like deleted scripts while execution!
                    invalid.deleteOnExit();
                }
            }
        }
        // latest downloaded release
        yacyVersion dflt = (downloadedReleases.isEmpty()) ? null : downloadedReleases.last();
        // check if there are any downloaded releases and if there are enable the update buttons
        prop.put("candeploy_downloadsAvailable", (downloadedReleases.isEmpty()) ? "0" : "1");
        prop.put("candeploy_deployenabled_buttonsActive", (downloadedReleases.isEmpty() || devenvironment) ? "0" : "1");
   
        int relcount = 0;
        for(final yacyRelease release : downloadedReleases) {
            prop.put("candeploy_downloadedreleases_" + relcount + "_name", ((release.isMainRelease()) ? "main" : "dev") + " " + release.getReleaseNr() + "/" + release.getSvn());
            prop.put("candeploy_downloadedreleases_" + relcount + "_signature", (release.getSignatureFile().exists() ? "1" : "0"));
            prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.getName());
            prop.put("candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
            relcount++;
        }
        prop.put("candeploy_downloadedreleases", relcount);

        // list remotely available releases
        final yacyRelease.DevAndMainVersions releasess = yacyRelease.allReleases(false, false);
        relcount = 0;
       
        // main
        final Set<yacyRelease> remoteMainReleases = releasess.main;
        remoteMainReleases.removeAll(downloadedReleases);
        for (final yacyRelease release : remoteMainReleases) {
            prop.put("candeploy_availreleases_" + relcount + "_name", ((release.isMainRelease()) ? "main" : "dev") + " " + release.getReleaseNr() + "/" + release.getSvn());
            prop.put("candeploy_availreleases_" + relcount + "_url", release.getUrl().toString());
            prop.put("candeploy_availreleases_" + relcount + "_signatures", (release.getPublicKey()!=null?"1":"0"));
            prop.put("candeploy_availreleases_" + relcount + "_selected", "0");
            relcount++;
        }
       
        // dev
        dflt = (releasess.dev.isEmpty()) ? null : releasess.dev.last();
        final Set<yacyRelease> remoteDevReleases = releasess.dev;
        remoteDevReleases.removeAll(downloadedReleases);
        for (final yacyRelease release : remoteDevReleases) {
            prop.put("candeploy_availreleases_" + relcount + "_name", ((release.isMainRelease()) ? "main" : "dev") + " " + release.getReleaseNr() + "/" + release.getSvn());
            prop.put("candeploy_availreleases_" + relcount + "_url", release.getUrl().toString());
            prop.put("candeploy_availreleases_" + relcount + "_signatures", (release.getPublicKey()!=null?"1":"0"));
            prop.put("candeploy_availreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
            relcount++;
        }
        prop.put("candeploy_availreleases", relcount);

        // properties for automated system update
        prop.put("candeploy_manualUpdateChecked", ("manual".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
        prop.put("candeploy_autoUpdateChecked", ("auto".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
        prop.put("candeploy_cycle", sb.getConfigLong("update.cycle", 168));
        prop.putHTML("candeploy_blacklist", sb.getConfig("update.blacklist", ""));
        prop.put("candeploy_releaseTypeMainChecked", ("any".equals(sb.getConfig("update.concept", "any"))) ? "0" : "1");
        prop.put("candeploy_releaseTypeAnyChecked", ("any".equals(sb.getConfig("update.concept", "any"))) ? "1" : "0");
        prop.put("candeploy_lastlookup", (sb.getConfigLong("update.time.lookup", 0) == 0) ? "0" : "1");
        prop.put("candeploy_lastlookup_time", new Date(sb.getConfigLong("update.time.lookup", 0)).toString());
        prop.put("candeploy_lastdownload", (sb.getConfigLong("update.time.download", 0) == 0) ? "0" : "1");
        prop.put("candeploy_lastdownload_time", new Date(sb.getConfigLong("update.time.download", 0)).toString());
        prop.put("candeploy_lastdeploy", (sb.getConfigLong("update.time.deploy", 0) == 0) ? "0" : "1");
        prop.put("candeploy_lastdeploy_time", new Date(sb.getConfigLong("update.time.deploy", 0)).toString());
        prop.put("candeploy_onlySignedFiles", ("1".equals(sb.getConfig("update.onlySignedFiles", "1"))) ? "1" : "0");
       
        /*
        if ((adminaccess) && (yacyVersion.latestRelease >= (thisVersion+0.01))) { // only new Versions(not new SVN)
            if ((yacyVersion.latestMainRelease != null) ||
                (yacyVersion.latestDevRelease != null)) {
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
    }
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        File defaultSettingsFile = new File(sb.getAppPath(), "defaults/yacy.init");
       
        // get segment
        Segment indexSegment = null;
        if (post != null && post.containsKey("segment")) {
            String segmentName = post.get("segment");
            if (sb.indexSegments.segmentExist(segmentName)) {
                indexSegment = sb.indexSegments.segment(segmentName);
            }
        } else {
            // take default segment
            indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }
       
        if(post != null) {
          if(post.containsKey("defaultFile")){
              // TODO check file-path!
              final File value = new File(sb.getAppPath(), post.get("defaultFile", "defaults/yacy.init"));
              // check if value is readable file
              if(value.exists() && value.isFile() && value.canRead()) {
                  defaultSettingsFile = value;
              }
          }
            if (post.containsKey("Xmx")) {
                int xmx = post.getInt("Xmx", 500); // default maximum heap size
                if (OS.isWin32) xmx = Math.min(2000, xmx);
                int xms = xmx / 4;
              sb.setConfig("javastart_Xmx", "Xmx" + xmx + "m");
              sb.setConfig("javastart_Xms", "Xms" + xms + "m");
              prop.put("setStartupCommit", "1");
            }
            if(post.containsKey("diskFree")) {
              sb.setConfig(SwitchboardConstants.DISK_FREE, post.getInt("diskFree", 3000));
            }
            if(post.containsKey("diskFreeHardlimit")) {
              sb.setConfig(SwitchboardConstants.DISK_FREE_HARDLIMIT, post.getInt("diskFreeHardlimit", 1000));
            }
            if(post.containsKey("memoryAcceptDHT")) {
              sb.setConfig(SwitchboardConstants.MEMORY_ACCEPTDHT, post.getInt("memoryAcceptDHT", 50));
            }
            if(post.containsKey("resetObserver")) {
              MemoryControl.setDHTallowed();
            }
        }
        final Map<String, String> defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : FileUtils.loadMap(defaultSettingsFile);
        Iterator<String> threads = sb.threadNames();
        String threadName;
        BusyThread thread;
       
        final boolean xml = (header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml");
        prop.setLocalized(!xml);
       
        // calculate totals
        long blocktime_total = 0, sleeptime_total = 0, exectime_total = 0;
        while (threads.hasNext()) {
            threadName = threads.next();
            thread = sb.getThread(threadName);
            blocktime_total += thread.getBlockTime();
            sleeptime_total += thread.getSleepTime();
            exectime_total += thread.getExecTime();
        }  
        if (blocktime_total == 0) blocktime_total = 1;
        if (sleeptime_total == 0) sleeptime_total = 1;
        if (exectime_total == 0) exectime_total = 1;
       
        // set templates for latest news from the threads
        long blocktime, sleeptime, exectime;
        long idlesleep, busysleep, memuse, memprereq;
        int queuesize;
        threads = sb.threadNames();
        int c = 0;
        long idleCycles, busyCycles, memshortageCycles;
        // set profile?
        final double multiplier = (post != null) && post.containsKey("profileSpeed") ? 100.0 / post.getFloat("profileSpeed", 100.0f) : 1.0;
        final boolean setProfile = (post != null && post.containsKey("submitdefault"));
        final boolean setDelay = (post != null) && (post.containsKey("submitdelay"));
        // save used settings file to config
        if (setProfile && post != null){
          sb.setConfig("performanceProfile", post.get("defaultFile", "defaults/yacy.init"));
          sb.setConfig("performanceSpeed", post.getInt("profileSpeed", 100));
        }
       
        while (threads.hasNext()) {
            threadName = threads.next();
            thread = sb.getThread(threadName);
           
            // set values to templates
            prop.put("table_" + c + "_threadname", threadName);

      prop.putHTML("table_" + c + "_hasurl_shortdescr", thread.getShortDescription());
      if(thread.getMonitorURL() == null) {
        prop.put("table_"+c+"_hasurl", "0");
      }else{
        prop.put("table_"+c+"_hasurl", "1");
        prop.put("table_" + c + "_hasurl_url", thread.getMonitorURL());
      }
            prop.putHTML("table_" + c + "_longdescr", thread.getLongDescription());
            queuesize = thread.getJobCount();
            prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : Formatter.number(queuesize, !xml));
           
            blocktime = thread.getBlockTime();
            sleeptime = thread.getSleepTime();
            exectime = thread.getExecTime();
            memuse = thread.getMemoryUse();
            idleCycles = thread.getIdleCycles();
            busyCycles = thread.getBusyCycles();
            memshortageCycles = thread.getOutOfMemoryCycles();
            prop.putNum("table_" + c + "_blocktime", blocktime / 1000);
            prop.putNum("table_" + c + "_blockpercent", 100 * blocktime / blocktime_total);
            prop.putNum("table_" + c + "_sleeptime", sleeptime / 1000);
            prop.putNum("table_" + c + "_sleeppercent", 100 * sleeptime / sleeptime_total);
            prop.putNum("table_" + c + "_exectime", exectime / 1000);
            prop.putNum("table_" + c + "_execpercent", 100 * exectime / exectime_total);
            prop.putNum("table_" + c + "_totalcycles", idleCycles + busyCycles + memshortageCycles);
            prop.putNum("table_" + c + "_idlecycles", idleCycles);
            prop.putNum("table_" + c + "_busycycles", busyCycles);
            prop.putNum("table_" + c + "_memscycles", memshortageCycles);
            prop.putNum("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? -1 : sleeptime / (idleCycles + busyCycles));
            prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles);
            prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024);
           
            // load with old values
            idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000);
            busysleep = sb.getConfigLong(threadName + "_busysleep",   100);
            memprereq = sb.getConfigLong(threadName + "_memprereq",     0);
            if (setDelay && post != null) {
                // load with new values
                idlesleep = post.getLong(threadName + "_idlesleep", idlesleep);
                busysleep = post.getLong(threadName + "_busysleep", busysleep);
                memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024;
                if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
                   
                // check values to prevent short-cut loops
                if (idlesleep < 1000) idlesleep = 1000;
                if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
               
                sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
                idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
                busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
            }
            if (setProfile) {
                if (threadName.equals(SwitchboardConstants.PEER_PING) ||
                    threadName.equals(SwitchboardConstants.SEED_UPLOAD) ||
                    threadName.equals(SwitchboardConstants.CLEANUP)) {
                    /* do not change any values */
                } else {
                    // load with new values
                    idlesleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_idlesleep"), String.valueOf(idlesleep))) * multiplier);
                    busysleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_busysleep"), String.valueOf(busysleep))) * multiplier);
                    //memprereq = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_memprereq"), String.valueOf(memprereq))) * multiplier);

                    // check values to prevent short-cut loops
                    if (idlesleep < 1000) idlesleep = 1000;
                    if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
                    //if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50;
                    sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
                }
            }
            prop.put("table_" + c + "_idlesleep", idlesleep);
            prop.put("table_" + c + "_busysleep", busysleep);
            prop.put("table_" + c + "_memprereq", memprereq / 1024);
            // disallow setting of memprereq for indexer to prevent db from throwing OOMs
            prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0");
            prop.put("table_" + c + "_recommendation", threadName.endsWith("_indexing") ? "1" : "0");
            prop.putNum("table_" + c + "_recommendation_value", threadName.endsWith("_indexing") ? (indexSegment.termIndex().minMem() / 1024) : 0);
            c++;
        }
        prop.put("table", c);
       
        // performance profiles
        c = 0;
        final String usedfile = sb.getConfig("performanceProfile", "defaults/yacy.init");
        for(final String filename: performanceProfiles.keySet()) {
            prop.put("profile_" + c + "_filename", filename);
            prop.put("profile_" + c + "_description", performanceProfiles.get(filename));
            prop.put("profile_" + c + "_used", usedfile.equalsIgnoreCase(filename) ? "1" : "0");
            c++;
        }
        prop.put("profile", c);
       
        c = 0;
        final int[] speedValues = {200,150,100,50,25,10};
        final int usedspeed = sb.getConfigInt("performanceSpeed", 100);
        for(final int speed: speedValues){
          prop.put("speed_" + c + "_value", speed);
          prop.put("speed_" + c + "_label", speed + " %");
          prop.put("speed_" + c + "_used", (speed == usedspeed) ? "1" : "0");
          c++;
        }
        prop.put("speed", c);
       
        if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
            final int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000);
            sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));
            indexSegment.termIndex().setBufferMaxWordCount(wordCacheMaxCount);
        }
       
        if ((post != null) && (post.containsKey("poolConfig"))) {
           
            /*
             * configuring the crawler pool
             */
            // get the current crawler pool configuration
            int maxBusy = post.getInt("Crawler Pool_maxActive", 8);
           
            // storing the new values into configfile
            sb.setConfig(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX,maxBusy);
            //switchboard.setConfig("crawler.MinIdleThreads",minIdle);
           
            /*
             * configuring the http pool
             */
            final WorkflowThread httpd = sb.getThread("10_httpd");
            try {
                maxBusy = post.getInt("httpd Session Pool_maxActive", 8);
            } catch (final NumberFormatException e) {
                maxBusy = 8;
            }

            ((serverCore)httpd).setMaxSessionCount(maxBusy);   
           
            // storing the new values into configfile
            sb.setConfig("httpdMaxBusySessions",maxBusy);

        }       
       
        if ((post != null) && (post.containsKey("PrioritySubmit"))) {
          sb.setConfig("javastart_priority",post.get("YaCyPriority","0"));
        }
       
        if ((post != null) && (post.containsKey("onlineCautionSubmit"))) {
            sb.setConfig(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000)));
            sb.setConfig(SwitchboardConstants.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000)));
            sb.setConfig(SwitchboardConstants.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000)));
        }
       
        if ((post != null) && (post.containsKey("minimumDeltaSubmit"))) {
            final long minimumLocalDelta = post.getLong("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta());
            final long minimumGlobalDelta = post.getLong("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta());
            sb.setConfig("minimumLocalDelta", minimumLocalDelta);
            sb.setConfig("minimumGlobalDelta", minimumGlobalDelta);
            sb.crawlQueues.noticeURL.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta);
        }
       
        // delta settings
        prop.put("minimumLocalDelta", sb.crawlQueues.noticeURL.getMinimumLocalDelta());
        prop.put("minimumGlobalDelta", sb.crawlQueues.noticeURL.getMinimumGlobalDelta());
       
        // table cache settings
        prop.putNum("urlCacheSize", indexSegment.urlMetadata().writeCacheSize()); 
        prop.putNum("wordCacheSize", indexSegment.termIndex().getBufferSize());
        prop.putNum("wordCacheSizeKBytes", indexSegment.termIndex().getBufferSizeBytes()/1024);
        prop.putNum("maxURLinCache", indexSegment.termIndex().getBufferMaxReferences());
        prop.putNum("maxAgeOfCache", indexSegment.termIndex().getBufferMaxAge() / 1000 / 60); // minutes
        prop.putNum("minAgeOfCache", indexSegment.termIndex().getBufferMinAge() / 1000 / 60); // minutes
        prop.putNum("maxWaitingWordFlush", sb.getConfigLong("maxWaitingWordFlush", 180));
        prop.put("wordCacheMaxCount", sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000));
        prop.put("crawlPauseProxy", sb.getConfigLong(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, 30000));
        prop.put("crawlPauseLocalsearch", sb.getConfigLong(SwitchboardConstants.LOCALSEACH_ONLINE_CAUTION_DELAY, 30000));
        prop.put("crawlPauseRemotesearch", sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_ONLINE_CAUTION_DELAY, 30000));
        prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - sb.proxyLastAccess) / 1000);
        prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - sb.localSearchLastAccess) / 1000);
        prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - sb.remoteSearchLastAccess) / 1000);
       
        // table thread pool settings
        prop.put("pool_0_name","Crawler Pool");
        prop.put("pool_0_maxActive", sb.getConfigLong("crawler.MaxActiveThreads", 0));
        prop.put("pool_0_numActive",sb.crawlQueues.workerSize());
       
        final WorkflowThread httpd = sb.getThread("10_httpd");
        prop.put("pool_1_name", "httpd Session Pool");
        prop.put("pool_1_maxActive", ((serverCore)httpd).getMaxSessionCount());
        prop.put("pool_1_numActive", ((serverCore)httpd).getJobCount());
       
        prop.put("pool", "2");
       
        final long curr_prio = sb.getConfigLong("javastart_priority",0);
        prop.put("priority_normal",(curr_prio == 0) ? "1" : "0");
        prop.put("priority_below",(curr_prio == 10) ? "1" : "0");
        prop.put("priority_low",(curr_prio == 20) ? "1" : "0");
       
        // parse initialization memory settings
        final String Xmx = sb.getConfig("javastart_Xmx", "Xmx500m").substring(3);
        prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1));
        final String Xms = sb.getConfig("javastart_Xms", "Xms500m").substring(3);
        prop.put("Xms", Xms.substring(0, Xms.length() - 1));
       
        final long diskFree = sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000L);
        final long diskFreeHardlimit = sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 1000L);
        final long memoryAcceptDHT = sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 50000L);
        final boolean observerTrigger = !MemoryControl.getDHTallowed();
        prop.put("diskFree", diskFree);
        prop.put("diskFreeHardlimit", diskFreeHardlimit);
        prop.put("memoryAcceptDHT", memoryAcceptDHT);
        if(observerTrigger) prop.put("observerTrigger", "1");
       
        // return rewrite values for templates
        return prop;
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;

        final serverObjects prop = new serverObjects();

        // set default values
        prop.put("urlstring", "");
        prop.put("urlhash", "");
        prop.put("result", "");
        String segmentName = sb.getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default");
        int i = 0;
        for (final String s: sb.indexSegments.segmentNames()) {
            prop.put("segments_" + i + "_name", s);
            prop.put("segments_" + i + "_selected", (segmentName.equals(s)) ? 1 : 0);
            i++;
        }
        Segment segment = sb.indexSegments.segment(segmentName);
        prop.put("segments", i);
        prop.putNum("ucount", segment.urlMetadata().size());
        prop.put("otherHosts", "");
        prop.put("genUrlProfile", 0);
        prop.put("statistics", 1);
        prop.put("statistics_lines", 100);
        prop.put("statisticslines", 0);
        prop.put("reload", 0);

        // do segment selection
        if (post != null && post.containsKey("segment")) {
            // default values
            segmentName = post.get("segment", segmentName).trim();
            i= 0;
            for (final String s: sb.indexSegments.segmentNames()) {
                prop.put("segments_" + i + "_name", s);
                prop.put("segments_" + i + "_selected", (segmentName.equals(s)) ? 1 : 0);
                i++;
            }
            prop.put("segments", i);
            segment = sb.indexSegments.segment(segmentName);
        }

        // show export messages
        final MetadataRepository.Export export = segment.urlMetadata().export();
        if ((export != null) && (export.isAlive())) {
          // there is currently a running export
            prop.put("lurlexport", 2);
            prop.put("lurlexportfinished", 0);
        prop.put("lurlexporterror", 0);
        prop.put("lurlexport_exportfile", export.file().toString());
            prop.put("lurlexport_urlcount", export.count());
            prop.put("reload", 1);
        } else {
            prop.put("lurlexport", 1);
            prop.put("lurlexport_exportfile", sb.getDataPath() + "/DATA/EXPORT/" + GenericFormatter.SHORT_SECOND_FORMATTER.format());
            if (export == null) {
                // there has never been an export
                prop.put("lurlexportfinished", 0);
                prop.put("lurlexporterror", 0);
            } else {
                // an export was running but has finished
                prop.put("lurlexportfinished", 1);
                prop.put("lurlexportfinished_exportfile", export.file().toString());
                prop.put("lurlexportfinished_urlcount", export.count());
                if (export.failed() == null) {
                    prop.put("lurlexporterror", 0);
                } else {
                    prop.put("lurlexporterror", 1);
                    prop.put("lurlexporterror_exportfile", export.file().toString());
                    prop.put("lurlexporterror_exportfailmsg", export.failed());
                }
            }
        }

        if (post == null || env == null) {
            return prop; // nothing to do
        }

        // post values that are set on numerous input fields with same name
        String urlstring = post.get("urlstring", "").trim();
        String urlhash = post.get("urlhash", "").trim();

        if (!urlstring.startsWith("http://") &&
            !urlstring.startsWith("https://") &&
            !urlstring.startsWith("ftp://") &&
            !urlstring.startsWith("smb://") &&
            !urlstring.startsWith("file://")) { urlstring = "http://" + urlstring; }

        prop.putHTML("urlstring", urlstring);
        prop.putHTML("urlhash", urlhash);
        prop.put("result", " ");

        if (post.containsKey("urlhashdeleteall")) {
            i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, CacheStrategy.IFEXIST);
            prop.put("result", "Deleted URL and " + i + " references from " + i + " word indexes.");
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        if (post.containsKey("urlhashdelete")) {
            final URIMetadataRow entry = segment.urlMetadata().load(ASCII.getBytes(urlhash));
            if (entry == null) {
                prop.putHTML("result", "No Entry for URL hash " + urlhash + "; nothing deleted.");
            } else {
                urlstring = entry.metadata().url().toNormalform(false, true);
                prop.put("urlstring", "");
                sb.urlRemove(segment, urlhash.getBytes());
                prop.putHTML("result", "Removed URL " + urlstring);
            }
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        if (post.containsKey("urldelete")) {
            try {
                urlhash = ASCII.String((new DigestURI(urlstring)).hash());
            } catch (final MalformedURLException e) {
                urlhash = null;
            }
            if ((urlhash == null) || (urlstring == null)) {
                prop.put("result", "No input given; nothing deleted.");
            } else {
                sb.urlRemove(segment, urlhash.getBytes());
                prop.putHTML("result", "Removed URL " + urlstring);
            }
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        if (post.containsKey("urlstringsearch")) {
            try {
                final DigestURI url = new DigestURI(urlstring);
                urlhash = ASCII.String(url.hash());
                prop.put("urlhash", urlhash);
                final URIMetadataRow entry = segment.urlMetadata().load(ASCII.getBytes(urlhash));
                if (entry == null) {
                    prop.putHTML("result", "No Entry for URL " + url.toNormalform(true, true));
                    prop.putHTML("urlstring", urlstring);
                    prop.put("urlhash", "");
                } else {
                    prop.putAll(genUrlProfile(segment, entry, urlhash));
                    prop.put("statistics", 0);
                }
            } catch (final MalformedURLException e) {
                prop.putHTML("result", "bad url: " + urlstring);
                prop.put("urlhash", "");
            }
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        if (post.containsKey("urlhashsearch")) {
            final URIMetadataRow entry = segment.urlMetadata().load(ASCII.getBytes(urlhash));
            if (entry == null) {
                prop.putHTML("result", "No Entry for URL hash " + urlhash);
            } else {
                prop.putHTML("urlstring", entry.metadata().url().toNormalform(false, true));
                prop.putAll(genUrlProfile(segment, entry, urlhash));
                prop.put("statistics", 0);
            }
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        // generate list
        if (post.containsKey("urlhashsimilar")) {
            try {
                final Iterator<URIMetadataRow> entryIt = new RotateIterator<URIMetadataRow>(segment.urlMetadata().entries(true, urlhash), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), segment.termIndex().sizesMax());
                final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />");
                URIMetadataRow entry;
                i = 0;
                int rows = 0, cols = 0;
                prop.put("urlhashsimilar", "1");
                while (entryIt.hasNext() && i < 256) {
                    entry = entryIt.next();
                    if (entry == null) break;
                    prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash()));
                    cols++;
                    if (cols==8) {
                        prop.put("urlhashsimilar_rows_"+rows+"_cols", cols);
                        cols = 0;
                        rows++;
                    }
                    i++;
                }
                prop.put("statistics", 0);
                prop.put("urlhashsimilar_rows", rows);
                prop.put("result", result.toString());
            } catch (final IOException e) {
                prop.putHTML("result", "No Entries for URL hash " + urlhash);
            }
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        if (post.containsKey("lurlexport")) {
            // parse format
            int format = 0;
            final String fname = post.get("format", "url-text");
            final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain
            if (fname.endsWith("text")) format = 0;
            if (fname.endsWith("html")) format = 1;
            if (fname.endsWith("rss")) format = 2;

            // extend export file name
      String s = post.get("exportfile", "");
      if (s.indexOf('.') < 0) {
        if (format == 0) s = s + ".txt";
        if (format == 1) s = s + ".html";
        if (format == 2) s = s + ".xml";
      }
          final File f = new File(s);
      f.getParentFile().mkdirs();
      final String filter = post.get("exportfilter", ".*");
      final MetadataRepository.Export running = segment.urlMetadata().export(f, filter, null, format, dom);

      prop.put("lurlexport_exportfile", s);
      prop.put("lurlexport_urlcount", running.count());
      if ((running != null) && (running.failed() == null)) {
        prop.put("lurlexport", 2);
      }
      prop.put("reload", 1);
        }

        if (post.containsKey("deletedomain")) {
            final String hp = post.get("hashpart");
            try {
                segment.urlMetadata().deleteDomain(hp);
            } catch (final IOException e) {
                // TODO Auto-generated catch block
                Log.logException(e);
            }
            // trigger the loading of the table
            post.put("statistics", "");
            prop.put("reload", 0);
        }

        if (post.containsKey("statistics")) {
            final int count = post.getInt("lines", 100);
            Iterator<MetadataRepository.HostStat> statsiter;
            prop.put("statistics_lines", count);
            int cnt = 0;
            try {
                final MetadataRepository metadata = segment.urlMetadata();
                statsiter = metadata.statistics(count, metadata.urlSampleScores(metadata.domainSampleCollector()));
                boolean dark = true;
                MetadataRepository.HostStat hs;
                while (statsiter.hasNext() && cnt < count) {
                    hs = statsiter.next();
                    prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0");
                    prop.put("statisticslines_domains_" + cnt + "_domain", hs.hostname + ((hs.port == 80) ? "" : ":" + hs.port));
                    prop.put("statisticslines_domains_" + cnt + "lines", count);
                    prop.put("statisticslines_domains_" + cnt + "_hashpart", hs.hosthash);
                    prop.put("statisticslines_domains_" + cnt + "_count", hs.count);
                    dark = !dark;
                    cnt++;
                }
            } catch (final IOException e) {
                Log.logException(e);
            }
            prop.put("statisticslines_domains", cnt);
            prop.put("statisticslines", 1);
            prop.put("lurlexport", 0);
            prop.put("reload", 0);
        }

        // insert constants
        prop.putNum("ucount", segment.urlMetadata().size());
        // return rewrite properties
        return prop;
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
    private static boolean col = true;

    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();

        final String eventID = post.get("eventID", "");
        final boolean authenticated = sb.adminAuthenticated(header) >= 2;
        final int item = post.getInt("item", -1);
        final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
        final String path = header.get(HeaderFramework.CONNECTION_PROP_PATH);
        final boolean isHtml = path.endsWith(".html");
        final boolean isJson = path.endsWith(".json");

        // default settings for blank item
        prop.put("content", "0");
        prop.put("rss", "0");
        prop.put("references", "0");
        prop.put("rssreferences", "0");
        prop.put("dynamic", "0");

        // find search event
        final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
        if (theSearch == null) {
            // the event does not exist, show empty page
            return prop;
        }
        final QueryParams theQuery = theSearch.getQuery();

        // dynamically update count values
        final int totalcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount();
        final int offset = theQuery.neededResults() - theQuery.displayResults() + 1;
        prop.put("offset", offset);
        prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount)));
        prop.put("itemsperpage", Formatter.number(theQuery.itemsPerPage));
        prop.put("totalcount", Formatter.number(totalcount, true));
        prop.put("localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true));
        prop.put("localMissCount", Formatter.number(theSearch.getRankingResult().getMissCount(), true));
        prop.put("remoteResourceSize", Formatter.number(theSearch.getRankingResult().getRemoteResourceSize(), true));
        prop.put("remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
        prop.put("remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));

        final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
        if (theQuery.contentdom == ContentDomain.TEXT) {
            // text search

            // generate result object
            final ResultEntry result = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000);
            if (result == null) return prop; // no content

            final DigestURI resultURL = result.url();
            final int port = resultURL.getPort();
            DigestURI faviconURL = null;
            if ((isHtml || isJson) && !sb.isIntranetMode() && !resultURL.isLocal()) try {
                faviconURL = new DigestURI(resultURL.getProtocol() + "://" + resultURL.getHost() + ((port != -1) ? (":" + port) : "") + "/favicon.ico");
            } catch (final MalformedURLException e1) {
                Log.logException(e1);
                faviconURL = null;
            }

            prop.put("content", 1); // switch on specific content
            prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0);
            prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0);
            prop.put("content_showMetadata", sb.getConfigBool("search.result.show.metadata", true) ? 1 : 0);
            prop.put("content_showParser", sb.getConfigBool("search.result.show.parser", true) ? 1 : 0);
            prop.put("content_showPictures", sb.getConfigBool("search.result.show.pictures", true) ? 1 : 0);
            prop.put("content_authorized", authenticated ? "1" : "0");
            prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0");
            prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*");
            prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*");
            prop.put("content_authorized_urlhash", ASCII.String(result.hash()));
            final String resulthashString = ASCII.String(result.hash());
            prop.putHTML("content_title", result.title());
            prop.putXML("content_title-xml", result.title());
            prop.putJSON("content_title-json", result.title());
            prop.putHTML("content_link", result.urlstring());
            prop.putHTML("content_showPictures_link", result.urlstring());
            prop.putHTML("content_target", target);
            if (faviconURL != null && isHtml) sb.loader.loadIfNotExistBackground(faviconURL.toNormalform(true, false), 1024 * 1024 * 10);
            prop.putHTML("content_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // acquire license for favicon url loading
            prop.put("content_urlhash", resulthashString);
            prop.put("content_ranking", result.ranking);
            prop.put("content_showMetadata_urlhash", resulthashString);
            prop.put("content_showParser_urlhash", resulthashString);
            prop.put("content_urlhexhash", yacySeed.b64Hash2hexHash(resulthashString));
            prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), MAX_URL_LENGTH));
            prop.put("content_showDate_date", GenericFormatter.RFC1123_SHORT_FORMATTER.format(result.modified()));
            prop.put("content_date822", HeaderFramework.formatRFC1123(result.modified()));
            //prop.put("content_ybr", RankingProcess.ybr(result.hash()));
            prop.putHTML("content_size", Integer.toString(result.filesize())); // we don't use putNUM here because that number shall be usable as sorting key. To print the size, use 'sizename'
            prop.putHTML("content_sizename", sizename(result.filesize()));
            prop.putHTML("content_showSize_sizename", sizename(result.filesize()));
            prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost());
            prop.putHTML("content_file", resultURL.getFile());
            prop.putHTML("content_path", resultURL.getPath());
            prop.put("content_nl", (item == theQuery.offset) ? 0 : 1);
            prop.putHTML("content_publisher", result.publisher());
            prop.putHTML("content_creator", result.creator());// author
            prop.putHTML("content_subject", result.subject());
            final Set<String>[] query = theQuery.queryWords();
            final StringBuilder s = new StringBuilder(query[0].size() * 20);
            for (final String t: query[0]) {
                s.append('+').append(t);
            }
            final String words = (s.length() > 0) ? s.substring(1) : "";
            prop.putHTML("content_words", words);
            prop.putHTML("content_showParser_words", words);
            prop.putHTML("content_former", theQuery.queryString);
            prop.putHTML("content_showPictures_former", theQuery.queryString);
            final TextSnippet snippet = result.textSnippet();
            final String desc = (snippet == null) ? "" : snippet.getLineMarked(theQuery.fullqueryHashes);
            prop.put("content_description", desc);
            prop.putXML("content_description-xml", desc);
            prop.putJSON("content_description-json", desc);
            final SearchEvent.HeuristicResult heuristic = theSearch.getHeuristic(result.hash());
            if (heuristic == null) {
                prop.put("content_heuristic", 0);
            } else {
                if (heuristic.redundant) {
                    prop.put("content_heuristic", 1);
                } else {
                    prop.put("content_heuristic", 2);
                }
                prop.put("content_heuristic_name", heuristic.heuristicName);
            }
            EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false);
            final String ext = resultURL.getFileExtension().toLowerCase();
            if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) {
                final String license = sb.licensedURLs.aquireLicense(resultURL);
                prop.put("content_code", license);
            } else {
                prop.put("content_code", "");
            }
            if (result.lat() == 0.0f || result.lon() == 0.0f) {
                prop.put("content_loc", 0);
            } else {
                prop.put("content_loc", 1);
                prop.put("content_loc_lat", result.lat());
                prop.put("content_loc_lon", result.lon());
            }
            theQuery.transmitcount = item + 1;
            return prop;
        }

        if (theQuery.contentdom == ContentDomain.IMAGE) {
            // image search; shows thumbnails

            prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content
            final MediaSnippet ms = theSearch.result().oneImage(item);
            if (ms == null) {
                prop.put("content_item", "0");
            } else {
                final String license = sb.licensedURLs.aquireLicense(ms.href);
                sb.loader.loadIfNotExistBackground(ms.href.toNormalform(true, false), 1024 * 1024 * 10);
                prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
                prop.putHTML("content_item_href", ms.href.toNormalform(true, false));
                prop.put("content_item_code", license);
                prop.putHTML("content_item_name", shorten(ms.name, MAX_NAME_LENGTH));
                prop.put("content_item_mimetype", ms.mime);
                prop.put("content_item_fileSize", ms.fileSize);
                prop.put("content_item_width", ms.width);
                prop.put("content_item_height", ms.height);
                prop.put("content_item_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
                prop.put("content_item_urlhash", ASCII.String(ms.source.hash()));
                prop.put("content_item_source", ms.source.toNormalform(true, false));
                prop.putXML("content_item_source-xml", ms.source.toNormalform(true, false));
                prop.put("content_item_sourcedom", ms.source.getHost());
                prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1);
                prop.put("content_item", 1);
            }
            theQuery.transmitcount = item + 1;
            return prop;
        }

        if ((theQuery.contentdom == ContentDomain.AUDIO) ||
            (theQuery.contentdom == ContentDomain.VIDEO) ||
            (theQuery.contentdom == ContentDomain.APP)) {
            // any other media content

            // generate result object
            final ResultEntry result = theSearch.oneResult(item, 500);
            if (result == null) return prop; // no content

            prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content
            final List<MediaSnippet> media = result.mediaSnippets();
            if (item == 0) col = true;
            if (media != null) {
                int c = 0;
                for (final MediaSnippet ms : media) {
                    prop.putHTML("content_items_" + c + "_href", ms.href.toNormalform(true, false));
                    prop.putHTML("content_items_" + c + "_hrefshort", nxTools.shortenURLString(ms.href.toNormalform(true, false), MAX_URL_LENGTH));
                    prop.putHTML("content_items_" + c + "_target", target);
                    prop.putHTML("content_items_" + c + "_name", shorten(ms.name, MAX_NAME_LENGTH));
                    prop.put("content_items_" + c + "_col", (col) ? "0" : "1");
                    c++;
                    col = !col;
                }
                prop.put("content_items", c);
            } else {
                prop.put("content_items", "0");
            }
            theQuery.transmitcount = item + 1;
            return prop;
        }

View Full Code Here
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
   
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
        final serverObjects prop = new serverObjects();
        Segment segment = null;
        boolean html = post != null && post.containsKey("html");
        prop.setLocalized(html);
        if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
            segment = sb.indexSegments.segment(post.get("segment"));
        }
        if (segment == null) segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
       
        prop.put("rejected", "0");
        sb.updateMySeed();
        final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000);
        prop.putNum("ppm", sb.currentPPM());
        prop.putNum("qpm", sb.peers.mySeed().getQPM());
        prop.putNum("wordCacheSize", segment.termIndex().getBufferSize());
        prop.putNum("wordCacheMaxSize", cacheMaxSize);
   
        // crawl queues
        prop.putNum("localCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL).getJobCount());
        prop.putNum("limitCrawlSize", sb.crawlQueues.limitCrawlJobSize());
        prop.putNum("remoteCrawlSize", sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount());
        prop.putNum("loaderSize", sb.crawlQueues.workerSize());       
        prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
       
    // memory usage and system attributes
        prop.putNum("freeMemory", MemoryControl.free());
        prop.putNum("totalMemory", MemoryControl.total());
        prop.putNum("maxMemory", MemoryControl.maxMemory);
        prop.putNum("processors", WorkflowProcessor.availableCPU);

    // proxy traffic
    prop.put("trafficIn", ByteCount.getGlobalCount());
    prop.put("trafficProxy", ByteCount.getAccountCount(ByteCount.PROXY));
    prop.put("trafficCrawler", ByteCount.getAccountCount(ByteCount.CRAWLER));

        // return rewrite properties
        return prop;
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
   
    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        // return variable that accumulates replacements
        final Switchboard sb = (Switchboard) env;
       
        final serverObjects prop = new serverObjects();
        final Segment segment;
        boolean html = post != null && post.containsKey("html");
        prop.setLocalized(html);
        boolean authorized = sb.verifyAuthentication(header, false);
        if (post != null && post.containsKey("segment") && authorized) {
            segment = sb.indexSegments.segment(post.get("segment"));
        } else {
            segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
        }
       
        prop.put("dc_title", "");
        prop.put("dc_creator", "");
        prop.put("dc_description", "");
        prop.put("dc_subject", "");
        prop.put("dc_publisher", "");
        prop.put("dc_contributor", "");
        prop.put("dc_date", "");
        prop.put("dc_type", "");
        prop.put("dc_identifier", "");
        prop.put("dc_language", "");

        if (post == null) return prop;
       
        String urlstring = post.get("url", "").trim();
        String urlhash = post.get("urlhash", "").trim();
        if (urlstring.length() == 0 && urlhash.length() == 0) return prop;

        if (urlstring.length() > 0 && urlhash.length() == 0) {
            try {
                DigestURI url = new DigestURI(urlstring);
                urlhash = ASCII.String(url.hash());
            } catch (MalformedURLException e) {
                Log.logException(e);
            }
        }
        if (urlhash == null || urlhash.length() == 0) return prop;
       
        final URIMetadataRow entry = segment.urlMetadata().load(urlhash.getBytes());
        if (entry == null) return prop;

        final URIMetadataRow.Components metadata = entry.metadata();
        if (metadata.url() == null) {
            return prop;
        }
        final URIMetadataRow le = (entry.referrerHash() == null || entry.referrerHash().length != Word.commonHashLength) ? null : segment.urlMetadata().load(entry.referrerHash());
       
        prop.putXML("dc_title", metadata.dc_title());
        prop.putXML("dc_creator", metadata.dc_creator());
        prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate
        prop.putXML("dc_subject", metadata.dc_subject());
        prop.putXML("dc_publisher", metadata.dc_publisher());
        prop.putXML("dc_contributor", "");
        prop.putXML("dc_date", ISO8601Formatter.FORMATTER.format(entry.moddate()));
        prop.putXML("dc_type", String.valueOf(entry.doctype()));
        prop.putXML("dc_identifier", metadata.url().toNormalform(false, true));
        prop.putXML("dc_language", UTF8.String(entry.language()));
        prop.put("geo_lat", metadata.lat());
        prop.put("geo_long", metadata.lon());

        prop.put("yacy_urlhash", metadata.url().hash());
        prop.putXML("yacy_loaddate", entry.loaddate().toString());
        prop.putXML("yacy_referrer_hash", (le == null) ? "" : ASCII.String(le.hash()));
        prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true));
        prop.put("yacy_size", entry.size());
        prop.put("yacy_words",entry.wordCount());
       
        // return rewrite properties
        return prop;
    }
View Full Code Here
TOP

Related Classes of de.anomic.server.serverObjects

  • AccessTracker_p
  • add_ymark
  • all
  • BlacklistTest_p
  • Blog
  • Collage
  • ConfigAccounts_p
  • ConfigBasic
  • ConfigHTCache_p
  • ConfigLanguage_p
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.