Package com.commafeed.backend.model

Examples of com.commafeed.backend.model.Feed

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
public class Subscription implements Serializable {

  public static Subscription build(FeedSubscription subscription, String publicUrl, UnreadCount unreadCount) {
    Date now = new Date();
    FeedCategory category = subscription.getCategory();
    Feed feed = subscription.getFeed();
    Subscription sub = new Subscription();
    sub.setId(subscription.getId());
    sub.setName(subscription.getTitle());
    sub.setPosition(subscription.getPosition());
    sub.setMessage(feed.getMessage());
    sub.setErrorCount(feed.getErrorCount());
    sub.setFeedUrl(feed.getUrl());
    sub.setFeedLink(feed.getLink());
    sub.setIconUrl(FeedUtils.getFaviconUrl(subscription, publicUrl));
    sub.setLastRefresh(feed.getLastUpdated());
    sub.setNextRefresh((feed.getDisabledUntil() != null && feed.getDisabledUntil().before(now)) ? null : feed.getDisabledUntil());
    sub.setUnread(unreadCount.getUnreadCount());
    sub.setNewestItemTime(unreadCount.getNewestItemTime());
    sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
    sub.setFilter(subscription.getFilter());
    return sub;
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
    Preconditions.checkNotNull(req);
    Preconditions.checkNotNull(req.getId());

    FeedSubscription sub = feedSubscriptionDAO.findById(user, req.getId());
    if (sub != null) {
      Feed feed = sub.getFeed();
      queues.add(feed, true);
      return Response.ok().build();
    }
    return Response.ok(Status.NOT_FOUND).build();
  }
View Full Code Here
343
344
345
346
347
348
349
350
351
352
353
    Preconditions.checkNotNull(id);
    FeedSubscription subscription = feedSubscriptionDAO.findById(user, id);
    if (subscription == null) {
      return Response.status(Status.NOT_FOUND).build();
    }
    Feed feed = subscription.getFeed();
    byte[] icon = feedService.fetchFavicon(feed);

    ResponseBuilder builder = Response.ok(icon, "image/x-icon");

    CacheControl cacheControl = new CacheControl();
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
    }
    if (url.startsWith(pubUrl)) {
      throw new FeedSubscriptionException("Could not subscribe to a feed from this CommaFeed instance");
    }

    Feed feed = feedService.findOrCreate(url);

    FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
    if (sub == null) {
      sub = new FeedSubscription();
      sub.setFeed(feed);
View Full Code Here
84
85
86
87
88
89
90
91
92
  }

  public void refreshAll(User user) {
    List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
    for (FeedSubscription sub : subs) {
      Feed feed = sub.getFeed();
      queues.add(feed, true);
    }
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    }
  }

  public synchronized Feed findOrCreate(String url) {
    String normalized = FeedUtils.normalizeURL(url);
    Feed feed = feedDAO.findByUrl(normalized);
    if (feed == null) {
      feed = new Feed();
      feed.setUrl(url);
      feed.setNormalizedUrl(normalized);
      feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
      feed.setDisabledUntil(new Date(0));
      feedDAO.saveOrUpdate(feed);
    }
    return feed;
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    return query.orderBy(feed.disabledUntil.asc()).limit(count).distinct().list(feed);
  }

  public Feed findByUrl(String normalizedUrl) {
    List<Feed> feeds = newQuery().from(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).list(feed);
    Feed feed = Iterables.getFirst(feeds, null);
    if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) {
      return feed;
    }
    return null;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    }
  };

  public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
    FetchedFeed fetchedFeed = new FetchedFeed();
    Feed feed = fetchedFeed.getFeed();
    List<FeedEntry> entries = fetchedFeed.getEntries();

    try {
      String encoding = FeedUtils.guessEncoding(xml);
      String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
      if (xmlString == null) {
        throw new FeedException("Input string is null for url " + feedUrl);
      }
      xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
      InputSource source = new InputSource(new StringReader(xmlString));
      SyndFeed rss = new SyndFeedInput().build(source);
      handleForeignMarkup(rss);

      fetchedFeed.setTitle(rss.getTitle());
      feed.setPushHub(findHub(rss));
      feed.setPushTopic(findSelf(rss));
      feed.setUrl(feedUrl);
      feed.setLink(rss.getLink());
      List<SyndEntry> items = rss.getEntries();

      for (SyndEntry item : items) {
        FeedEntry entry = new FeedEntry();

        String guid = item.getUri();
        if (StringUtils.isBlank(guid)) {
          guid = item.getLink();
        }
        if (StringUtils.isBlank(guid)) {
          // no guid and no link, skip entry
          continue;
        }
        entry.setGuid(FeedUtils.truncate(guid, 2048));
        entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
        entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feed.getUrlAfterRedirect()), 2048));

        // if link is empty but guid is used as url
        if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
          entry.setUrl(entry.getGuid());
        }

        FeedEntryContent content = new FeedEntryContent();
        content.setContent(getContent(item));
        content.setTitle(getTitle(item));
        content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
        SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
        if (enclosure != null) {
          content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
          content.setEnclosureType(enclosure.getType());
        }
        entry.setContent(content);

        entries.add(entry);
      }
      Date lastEntryDate = null;
      Date publishedDate = validateDate(rss.getPublishedDate(), false);
      if (!entries.isEmpty()) {
        List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
        Long timestamp = sortedTimestamps.get(0);
        lastEntryDate = new Date(timestamp);
        publishedDate = (publishedDate == null || publishedDate.before(lastEntryDate)) ? lastEntryDate : publishedDate;
      }
      feed.setLastPublishedDate(publishedDate);
      feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
      feed.setLastEntryDate(lastEntryDate);

    } catch (Exception e) {
      throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
    }
    return fetchedFeed;
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    // set the disabledDate as we use it in feedDAO to decide what to refresh next. We also use a map to remove
    // duplicates.
    Map<Long, FeedRefreshContext> map = Maps.newLinkedHashMap();
    for (FeedRefreshContext context : contexts) {
      Feed feed = context.getFeed();
      feed.setDisabledUntil(DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()));
      map.put(feed.getId(), context);
    }

    // refill the queue
    takeQueue.addAll(map.values());

    // add feeds from the giveBack queue to the map, overriding duplicates
    int giveBackQueueSize = giveBackQueue.size();
    for (int i = 0; i < giveBackQueueSize; i++) {
      Feed feed = giveBackQueue.poll();
      map.put(feed.getId(), new FeedRefreshContext(feed, false));
    }

    // update all feeds in the database
    List<Feed> feeds = Lists.newArrayList();
    for (FeedRefreshContext context : map.values()) {
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
      return context.isUrgent();
    }
  }

  private void update(FeedRefreshContext context) {
    Feed feed = context.getFeed();
    int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
    Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
    try {
      String url = Optional.fromNullable(feed.getUrlAfterRedirect()).or(feed.getUrl());
      FetchedFeed fetchedFeed = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
          feed.getLastPublishedDate(), feed.getLastContentHash());
      // stops here if NotModifiedException or any other exception is thrown
      List<FeedEntry> entries = fetchedFeed.getEntries();

      if (config.getApplicationSettings().isHeavyLoad()) {
        disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
            .getAverageEntryInterval(), disabledUntil);
      }
      String urlAfterRedirect = fetchedFeed.getUrlAfterRedirect();
      if (StringUtils.equals(url, urlAfterRedirect)) {
        urlAfterRedirect = null;
      }
      feed.setUrlAfterRedirect(urlAfterRedirect);
      feed.setLink(fetchedFeed.getFeed().getLink());
      feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
      feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
      feed.setLastContentHash(fetchedFeed.getFeed().getLastContentHash());
      feed.setLastPublishedDate(fetchedFeed.getFeed().getLastPublishedDate());
      feed.setAverageEntryInterval(fetchedFeed.getFeed().getAverageEntryInterval());
      feed.setLastEntryDate(fetchedFeed.getFeed().getLastEntryDate());

      feed.setErrorCount(0);
      feed.setMessage(null);
      feed.setDisabledUntil(disabledUntil);

      handlePubSub(feed, fetchedFeed.getFeed());
      context.setEntries(entries);
      feedRefreshUpdater.updateFeed(context);

    } catch (NotModifiedException e) {
      log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());

      if (config.getApplicationSettings().isHeavyLoad()) {
        disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
      }
      feed.setErrorCount(0);
      feed.setMessage(e.getMessage());
      feed.setDisabledUntil(disabledUntil);

      queues.giveBack(feed);
    } catch (Exception e) {
      String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
      log.debug(e.getClass().getName() + " " + message, e);

      feed.setErrorCount(feed.getErrorCount() + 1);
      feed.setMessage(message);
      feed.setDisabledUntil(FeedUtils.buildDisabledUntil(feed.getErrorCount()));

      queues.giveBack(feed);
    }
  }
View Full Code Here
TOP

Related Classes of com.commafeed.backend.model.Feed

  • com.commafeed.backend.dao.FeedDAO
  • com.commafeed.backend.feed.FeedFetcher
  • com.commafeed.backend.feed.FeedParser
  • com.commafeed.backend.feed.FeedQueues
  • com.commafeed.backend.feed.FeedRefreshUpdater$EntryTask
  • com.commafeed.backend.feed.FeedRefreshWorker
  • com.commafeed.backend.service.FeedService
  • com.commafeed.backend.service.FeedSubscriptionService
  • com.commafeed.frontend.model.Subscription
  • com.commafeed.frontend.resource.FeedREST
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.