Examples of net.yacy.cora.document.RSSMessage

net.yacy.cora.document.RSSMessage

            } else {
                String firstHash = wordhashes.get(0);
                String lastHash = wordhashes.get(wordhashes.size() - 1);
                final long avdist = (FlatWordPartitionScheme.std.dhtDistance(firstHash.getBytes(), null, sb.peers.mySeed()) + FlatWordPartitionScheme.std.dhtDistance(lastHash.getBytes(), null, sb.peers.mySeed())) / 2;
                sb.getLog().logInfo("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "]/" + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName);
                yacyChannel.channels(yacyChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "]/" + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName, "", otherPeer.hash));
            }
            result = "ok";
            
            pause = (int) (sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() * 20000 / sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 100000)); // estimation of necessary pause time
        }

View Full Code Here

                String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss";
                BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
                SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null);
                
                // take the results and compute some locations
                RSSMessage message;
                loop: while ((message = results.poll(maximumTime, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {


                    // find all associated locations
                    Set<Location> locations = new HashSet<Location>();
                    StringBuilder words = new StringBuilder(120);
                    if (search_title) words.append(message.getTitle().trim()).append(space);
                    if (search_publisher) words.append(message.getCopyright().trim()).append(space);
                    if (search_creator) words.append(message.getAuthor().trim()).append(space);
                    String subject = "";
                    assert message != null;
                    assert message.getSubject() != null;
                    for (String s: message.getSubject()) subject += s.trim() + space;
                    if (search_subject) words.append(subject).append(space);
                    String[] wordlist = words.toString().trim().split(space);
                    for (String word: wordlist) if (word.length() >= 3) locations.addAll(LibraryProvider.geoLoc.find(word, true));
                    for (int i = 0; i < wordlist.length - 1; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1], true));
                    for (int i = 0; i < wordlist.length - 2; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1] + space + wordlist[i + 2], true));
                    
                    // add locations from metatag
                    if (metatag) {
                        if (message.getLat() != 0.0f && message.getLon() != 0.0f) {
                            locations.add(new Location(message.getLon(), message.getLat(), message.getTitle().trim()));
                        }
                    }
                    
                    for (Location location: locations) {
                        // write for all locations a point to this message
                        prop.put("kml_placemark_" + placemarkCounter + "_location", location.getName());
                        prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle());
                        prop.put("kml_placemark_" + placemarkCounter + "_author", message.getAuthor());
                        prop.put("kml_placemark_" + placemarkCounter + "_copyright", message.getCopyright());
                        prop.put("kml_placemark_" + placemarkCounter + "_subject", subject.trim());
                        prop.put("kml_placemark_" + placemarkCounter + "_description", message.getDescription());
                        prop.put("kml_placemark_" + placemarkCounter + "_date", message.getPubDate());
                        prop.putXML("kml_placemark_" + placemarkCounter + "_url", message.getLink());
                        prop.put("kml_placemark_" + placemarkCounter + "_pointname", location.getName());
                        prop.put("kml_placemark_" + placemarkCounter + "_lon", location.lon());
                        prop.put("kml_placemark_" + placemarkCounter + "_lat", location.lat());
                        placemarkCounter++;
                        if (placemarkCounter >= maximumRecords) break loop;

View Full Code Here


            // return rewrite properties
            final int more = sb.indexSegments.urlMetadata(Segments.Process.DHTIN).size() - sizeBefore;
            doublevalues = Integer.toString(received - more);
            yacyCore.log.logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
            yacyChannel.channels(yacyChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));
            if ((received - more) > 0) yacyCore.log.logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
            result = "ok";
        }


        prop.put("double", doublevalues);

View Full Code Here

                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
            yacyChannel.channels(yacyChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));


            // make event
            theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));


            // set statistic details of search result and find best result index set

View Full Code Here

            // filter out words that appear in bluelist
            theQuery.filterOut(Switchboard.blueList);


            // log
            Log.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + QueryParams.hashSet2hashString(theQuery.queryHashes) + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
            yacyChannel.channels(yacyChannel.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
            final long timestamp = System.currentTimeMillis();


            // create a new search event
            if (SearchEventCache.getEvent(theQuery.id(false)) == null) {
                theQuery.setOffset(0); // in case that this is a new search, always start without a offset

View Full Code Here

        // index all selected items: description only
        if (rss != null && post.containsKey("indexSelectedItemContent")) {
            final RSSFeed feed = rss.getFeed();
            loop: for (final Map.Entry<String, String> entry: post.entrySet()) {
                if (entry.getValue().startsWith("mark_")) try {
                    final RSSMessage message = feed.getMessage(entry.getValue().substring(5));
                    final DigestURI messageurl = new DigestURI(message.getLink());
                    if (RSSLoader.indexTriggered.containsKey(messageurl.hash())) continue loop;
                    if (sb.urlExists(Segments.Process.LOCALCRAWLING, messageurl.hash()) != null) continue loop;
                    sb.addToIndex(messageurl, null, null);
                    RSSLoader.indexTriggered.insertIfAbsent(messageurl.hash(), new Date());
                } catch (final IOException e) {
                    Log.logException(e);
                } catch (final Failure e) {
                    Log.logException(e);
                }
            }
        }


        if (rss != null && post.containsKey("indexAllItemContent")) {
            record_api = true;
            final RSSFeed feed = rss.getFeed();
            RSSLoader.indexAllRssFeed(sb, url, feed);
        }


        if (record_api && rss != null && rss.getFeed() != null && rss.getFeed().getChannel() != null) {
            // record API action
            RSSLoader.recordAPI(sb, post.get(WorkTables.TABLE_API_COL_APICALL_PK, null), url, rss.getFeed(), repeat_time, repeat_unit);
        }


        // show items from rss
        if (rss != null) {
            prop.put("showitems", 1);
            final RSSFeed feed = rss.getFeed();
            final RSSMessage channel = feed.getChannel();
            prop.putHTML("showitems_title", channel == null ? "" : channel.getTitle());
            String author = channel == null ? "" : channel.getAuthor();
            if (author == null || author.length() == 0) author = channel == null ? "" : channel.getCopyright();
            Date pubDate = channel == null ? null : channel.getPubDate();
            prop.putHTML("showitems_author", author == null ? "" : author);
            prop.putHTML("showitems_description", channel == null ? "" : channel.getDescription());
            prop.putHTML("showitems_language", channel == null ? "" : channel.getLanguage());
            prop.putHTML("showitems_date", (pubDate == null) ? "" : DateFormat.getDateTimeInstance().format(pubDate));
            prop.putHTML("showitems_ttl", channel == null ? "" : channel.getTTL());
            prop.putHTML("showitems_docs", channel == null ? "" : channel.getDocs());


            int i = 0;
            for (final Hit item: feed) {
                try {
                    final DigestURI messageurl = new DigestURI(item.getLink());

View Full Code Here


            // read the channel
            final RSSFeed feed = yacyChannel.channels(channel);
            if (feed == null || feed.isEmpty()) continue channelIteration;


            RSSMessage message = feed.getChannel();
            if (message != null) {
                prop.putXML("channel_title", message.getTitle());
                prop.putXML("channel_description", message.getDescription());
                prop.put("channel_pubDate", message.getPubDate());
            }
            while (messageMaxCount > 0 && !feed.isEmpty()) {
                message = feed.pollMessage();
                if (message == null) continue;


                // create RSS entry
                prop.putXML("item_" + messageCount + "_title", channelName + ": " + message.getTitle());
                prop.putXML("item_" + messageCount + "_description", message.getDescription());
                prop.putXML("item_" + messageCount + "_link", message.getLink());
                prop.put("item_" + messageCount + "_pubDate", message.getPubDate());
                prop.put("item_" + messageCount + "_guid", message.getGuid());
                messageCount++;
                messageMaxCount--;
            }
            if (messageMaxCount == 0) break channelIteration;
        }

View Full Code Here

        if (peer == null) return false;
        final boolean res = connectPeer(peer, direct);
        if (res) {
            // perform all actions if peer is effective new
            this.processPeerArrival(peer);
            yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
        }
        return res;
    }

View Full Code Here

        synchronized (seedDB) {
            if (!seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { disconnects++; }
            peer.put("dct", Long.toString(System.currentTimeMillis()));
            seedDB.addDisconnected(peer); // update info
        }
        yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
    }

View Full Code Here

        if (peer == null) return;
        // this is called only if the peer has junior status
        seedDB.addPotential(peer);
        // perform all actions
        processPeerArrival(peer);
        yacyChannel.channels(yacyChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", ""));
    }

View Full Code Here

0 1 2

TOP

Related Classes of net.yacy.cora.document.RSSMessage

de.anomic.search.Switchboard

de.anomic.yacy.yacyClient

de.anomic.yacy.yacyCore

de.anomic.yacy.yacyPeerActions

feed

Load_RSS_p

net.yacy.cora.services.federated.opensearch.SRURSSConnector

net.yacy.peers.Network

net.yacy.peers.PeerActions

net.yacy.peers.Protocol

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.