Package com.gwtplatform.crawlerservice.server.domain

Examples of com.gwtplatform.crawlerservice.server.domain.CachedPage

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
                List<Key<CachedPage>> keys = cachedPageDao.listKeysByProperty("url", url);
                Map<Key<CachedPage>, CachedPage> deprecatedPages = cachedPageDao.get(keys);

                Date currDate = new Date();

                CachedPage matchingPage = extractMatchingPage(deprecatedPages, currDate);
                cachedPageDao.deleteKeys(deprecatedPages.keySet());

                if (needToFetchPage(matchingPage, currDate, out)) {
                    CachedPage cachedPage = createPlaceholderPage(url, currDate);
                    StringBuilder renderedHtml = renderPage(url);
                    storeFetchedPage(cachedPage, renderedHtml);
                    out.println(renderedHtml.toString());
                }
            }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
     * @param url      The URL of the page for which to create a placeholder.
     * @param currDate The current date, to mark the page.
     * @return The newly created placeholder page.
     */
    private CachedPage createPlaceholderPage(String url, Date currDate) {
        CachedPage result = new CachedPage();
        result.setUrl(url);
        result.setFetchDate(currDate);
        result.setFetchInProgress(true);
        cachedPageDao.put(result);
        return result;
    }
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
     * @param currDate        The current date, to check for expiration.
     * @return The non-expired matching page if found, {@code null} otherwise.
     */
    private CachedPage extractMatchingPage(Map<Key<CachedPage>, CachedPage> deprecatedPages,
            Date currDate) {
        CachedPage matchingPage = findMostRecentPage(deprecatedPages);

        // Keep the matching page only if it has not expired
        if (matchingPage == null ||
                currDate.getTime() >
                        matchingPage.getFetchDate().getTime() + cachedPageTimeoutSec * 1000) {
            matchingPage = null;
        } else {
            deprecatedPages.remove(Key.create(CachedPage.class, matchingPage.getId()));
        }

        return matchingPage;
    }
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        return matchingPage;
    }

    private CachedPage findMostRecentPage(Map<Key<CachedPage>, CachedPage> pages) {
        CachedPage result = null;
        for (CachedPage page : pages.values()) {
            if (result == null ||
                    page.getFetchDate().after(result.getFetchDate())) {
                result = page;
            }
        }
        return result;
    }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
        List<Key<CachedPage>> keys = cachedPageDao.listKeysByProperty("url", url);
        Map<Key<CachedPage>, CachedPage> deprecatedPages = cachedPageDao.get(keys);

        Date currDate = new Date();

        CachedPage matchingPage = extractMatchingPage(deprecatedPages, currDate);
        cachedPageDao.deleteKeys(deprecatedPages.keySet());

        if (needToFetchPage(matchingPage, currDate, out)) {
          CachedPage cachedPage = createPlaceholderPage(url, currDate);
          StringBuilder renderedHtml = renderPage(url);
          storeFetchedPage(cachedPage, renderedHtml);
          out.println(renderedHtml.toString());
        }
      }
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
167
   * @param url The URL of the page for which to create a placeholder.
   * @param currDate The current date, to mark the page.
   * @return The newly created placeholder page.
   */
  private CachedPage createPlaceholderPage(String url, Date currDate) {
    CachedPage result = new CachedPage();
    result.setUrl(url);
    result.setFetchDate(currDate);
    result.setFetchInProgress(true);
    cachedPageDao.put(result);
    return result;
  }
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
   * @param currDate The current date, to check for expiration.
   * @return The non-expired matching page if found, {@code null} otherwise.
   */
  private CachedPage extractMatchingPage(Map<Key<CachedPage>, CachedPage> deprecatedPages,
      Date currDate) {
    CachedPage matchingPage = findMostRecentPage(deprecatedPages);

    // Keep the matching page only if it has not expired
    if (matchingPage == null ||
        currDate.getTime() >
            matchingPage.getFetchDate().getTime() + cachedPageTimeoutSec * 1000) {
      matchingPage = null;
    } else {
      deprecatedPages.remove(new Key<CachedPage>(CachedPage.class, matchingPage.getId()));
    }

    return matchingPage;
  }
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    return matchingPage;
  }

  private CachedPage findMostRecentPage(Map<Key<CachedPage>, CachedPage> pages) {
    CachedPage result = null;
    for (CachedPage page : pages.values()) {
      if (result == null ||
          page.getFetchDate().after(result.getFetchDate())) {
        result = page;
      }
    }
    return result;
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
                List<Key<CachedPage>> keys = cachedPageDao.listKeysByProperty("url", url);
                Map<Key<CachedPage>, CachedPage> deprecatedPages = cachedPageDao.get(keys);

                Date currDate = new Date();

                CachedPage matchingPage = extractMatchingPage(deprecatedPages, currDate);
                cachedPageDao.deleteKeys(deprecatedPages.keySet());

                if (needToFetchPage(matchingPage, currDate, out)) {
                    CachedPage cachedPage = createPlaceholderPage(url, currDate);
                    String renderedHtml = renderPage(url);
                    storeFetchedPage(cachedPage, renderedHtml);
                    out.println(renderedHtml);
                }
            }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
     * @param url      The URL of the page for which to create a placeholder.
     * @param currDate The current date, to mark the page.
     * @return The newly created placeholder page.
     */
    private CachedPage createPlaceholderPage(String url, Date currDate) {
        CachedPage result = new CachedPage();
        result.setUrl(url);
        result.setFetchDate(currDate);
        result.setFetchInProgress(true);
        cachedPageDao.put(result);
        return result;
    }
View Full Code Here
TOP

Related Classes of com.gwtplatform.crawlerservice.server.domain.CachedPage

  • com.gwtplatform.crawlerservice.server.CrawlServiceServlet
  • java.util.Date
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.