Examples of url()

@param url File URL or path relative to data directory @return File indicated by provided URL @see Files#url(File,String)
  • org.jeecgframework.core.annotation.config.AutoMenu.url()
  • org.jfrog.build.api.builder.BuildInfoBuilder.url()
    Sets the URL of the build @param url Build URL @return Builder instance
  • org.jfrog.build.api.builder.BuildInfoMavenBuilder.url()
    Sets the URL of the build @param url Build URL @return Builder instance
  • org.jsoup.Connection.url()
    Set the request URL to fetch. The protocol must be HTTP or HTTPS. @param url URL to connect to @return this Connection, for chaining
  • org.jsoup.Connection.Response.url()
  • org.mifosplatform.mix.data.NamespaceData.url()
  • org.onesocialweb.model.vcard4.VCard4Factory.url()
  • org.picketlink.idm.config.LDAPStoreConfigurationBuilder.url()
  • org.sgx.yuigwt.galleryexplorer.client.api.Module.url()
    gallery module url @return
  • org.stjs.testing.annotation.HTMLFixture.url()
  • play.mvc.Call.url()
    The request URL.

    • Examples of de.anomic.crawler.retrieval.Request.url()

    171
    172
    173
    174
    175
    176
    177
    178
    179
    180
    181
                        prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
                        prop.put("crawler-queue_list_"+showNum+"_depth", urle.depth());
                        prop.put("crawler-queue_list_"+showNum+"_modified", daydate(urle.appdate()) );
                        prop.putHTML("crawler-queue_list_"+showNum+"_anchor", urle.name());
                        prop.putHTML("crawler-queue_list_"+showNum+"_url", urle.url().toNormalform(false, true));
                        prop.put("crawler-queue_list_"+showNum+"_hash", urle.url().hash());
                        dark = !dark;
                        showNum++;
                    } else {
                        stackSize--;
                    }
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
                                    "client=____________",
                                    -1);
                   
                    // create RSS entry
                    prop.put("item_" + c + "_title", "");
                    prop.putXML("item_" + c + "_link", entry.url().toNormalform(true, false));
                    prop.putXML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false));
                    prop.putXML("item_" + c + "_description", entry.name());
                    prop.put("item_" + c + "_author", "");
                    prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.appdate()));
                    prop.put("item_" + c + "_guid", entry.url().hash());
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
                    prop.putXML("item_" + c + "_link", entry.url().toNormalform(true, false));
                    prop.putXML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false));
                    prop.putXML("item_" + c + "_description", entry.name());
                    prop.put("item_" + c + "_author", "");
                    prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.appdate()));
                    prop.put("item_" + c + "_guid", entry.url().hash());
                    c++;
                    maxCount--;
                }
                prop.put("item", c);
                prop.putXML("response", "ok");
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    415
    416
    417
    418
    419
    420
    421
    422
    423
    424
    425
                  return null;
                }
                // depending on the caching policy we need sleep time to avoid DoS-like situations
                sleeptime = (
                        profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY ||
                        (profileEntry.cacheStrategy() == CacheStrategy.IFEXIST && Cache.has(crawlEntry.url()))
                        ) ? 0 : Latency.waitingRemaining(crawlEntry.url(), this.myAgentIDs, this.minimumLocalDelta, this.minimumGlobalDelta); // this uses the robots.txt database and may cause a loading of robots.txt from the server

                assert Base64Order.enhancedCoder.equal(nexthash, rowEntry.getPrimaryKeyBytes()) : "result = " + ASCII.String(nexthash) + ", rowEntry.getPrimaryKeyBytes() = " + ASCII.String(rowEntry.getPrimaryKeyBytes());
                assert Base64Order.enhancedCoder.equal(nexthash, crawlEntry.url().hash()) : "result = " + ASCII.String(nexthash) + ", crawlEntry.url().hash() = " + ASCII.String(crawlEntry.url().hash());
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    416
    417
    418
    419
    420
    421
    422
    423
    424
    425
    426
                }
                // depending on the caching policy we need sleep time to avoid DoS-like situations
                sleeptime = (
                        profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY ||
                        (profileEntry.cacheStrategy() == CacheStrategy.IFEXIST && Cache.has(crawlEntry.url()))
                        ) ? 0 : Latency.waitingRemaining(crawlEntry.url(), this.myAgentIDs, this.minimumLocalDelta, this.minimumGlobalDelta); // this uses the robots.txt database and may cause a loading of robots.txt from the server

                assert Base64Order.enhancedCoder.equal(nexthash, rowEntry.getPrimaryKeyBytes()) : "result = " + ASCII.String(nexthash) + ", rowEntry.getPrimaryKeyBytes() = " + ASCII.String(rowEntry.getPrimaryKeyBytes());
                assert Base64Order.enhancedCoder.equal(nexthash, crawlEntry.url().hash()) : "result = " + ASCII.String(nexthash) + ", crawlEntry.url().hash() = " + ASCII.String(crawlEntry.url().hash());

                if (failhash != null && Base64Order.enhancedCoder.equal(failhash, nexthash)) break; // prevent endless loops
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    419
    420
    421
    422
    423
    424
    425
    426
    427
    428
    429
                        profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY ||
                        (profileEntry.cacheStrategy() == CacheStrategy.IFEXIST && Cache.has(crawlEntry.url()))
                        ) ? 0 : Latency.waitingRemaining(crawlEntry.url(), this.myAgentIDs, this.minimumLocalDelta, this.minimumGlobalDelta); // this uses the robots.txt database and may cause a loading of robots.txt from the server

                assert Base64Order.enhancedCoder.equal(nexthash, rowEntry.getPrimaryKeyBytes()) : "result = " + ASCII.String(nexthash) + ", rowEntry.getPrimaryKeyBytes() = " + ASCII.String(rowEntry.getPrimaryKeyBytes());
                assert Base64Order.enhancedCoder.equal(nexthash, crawlEntry.url().hash()) : "result = " + ASCII.String(nexthash) + ", crawlEntry.url().hash() = " + ASCII.String(crawlEntry.url().hash());

                if (failhash != null && Base64Order.enhancedCoder.equal(failhash, nexthash)) break; // prevent endless loops

                if (delay && sleeptime > 0 && this.domStackInitSize > 1) {
                    //System.out.println("*** putback: nexthash=" + nexthash + ", failhash="+failhash);
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    419
    420
    421
    422
    423
    424
    425
    426
    427
    428
    429
                        profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY ||
                        (profileEntry.cacheStrategy() == CacheStrategy.IFEXIST && Cache.has(crawlEntry.url()))
                        ) ? 0 : Latency.waitingRemaining(crawlEntry.url(), this.myAgentIDs, this.minimumLocalDelta, this.minimumGlobalDelta); // this uses the robots.txt database and may cause a loading of robots.txt from the server

                assert Base64Order.enhancedCoder.equal(nexthash, rowEntry.getPrimaryKeyBytes()) : "result = " + ASCII.String(nexthash) + ", rowEntry.getPrimaryKeyBytes() = " + ASCII.String(rowEntry.getPrimaryKeyBytes());
                assert Base64Order.enhancedCoder.equal(nexthash, crawlEntry.url().hash()) : "result = " + ASCII.String(nexthash) + ", crawlEntry.url().hash() = " + ASCII.String(crawlEntry.url().hash());

                if (failhash != null && Base64Order.enhancedCoder.equal(failhash, nexthash)) break; // prevent endless loops

                if (delay && sleeptime > 0 && this.domStackInitSize > 1) {
                    //System.out.println("*** putback: nexthash=" + nexthash + ", failhash="+failhash);
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    432
    433
    434
    435
    436
    437
    438
    439
    440
    441
    442
                        //System.out.println("*** delayed +=" + nexthash);
                        this.delayed.put(Long.valueOf(System.currentTimeMillis() + sleeptime + 1), nexthash);
                    }
                  try {
                            this.urlFileIndex.put(rowEntry);
                            String host = crawlEntry.url().getHost();
                            if (host == null) host = localhost;
                            this.domainStacks.remove(host);
                            failhash = nexthash;
                        } catch (final RowSpaceExceededException e) {
                            Log.logException(e);
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    445
    446
    447
    448
    449
    450
    451
    452
    453
    454
    455
                }
                break;
            }
            if (crawlEntry != null) {
                    if (this.ddc.size() > 10000 || MemoryControl.shortStatus()) this.ddc.clear();
                    try { this.ddc.put(crawlEntry.url().hash()); } catch (final RowSpaceExceededException e) {}
            }
          }
          if (crawlEntry == null) return null;

            if (delay && sleeptime > 0) {
    View Full Code Here

    Examples of de.anomic.crawler.retrieval.Request.url()

    455
    456
    457
    458
    459
    460
    461
    462
    463
    464
    465
            if (delay && sleeptime > 0) {
                // force a busy waiting here
                // in best case, this should never happen if the balancer works propertly
                // this is only to protection against the worst case, where the crawler could
                // behave in a DoS-manner
                Log.logInfo("BALANCER", "forcing crawl-delay of " + sleeptime + " milliseconds for " + crawlEntry.url().getHost() + ": " + Latency.waitingRemainingExplain(crawlEntry.url(), this.myAgentIDs, this.minimumLocalDelta, this.minimumGlobalDelta) + ", top.size() = " + this.top.size() + ", delayed.size() = " + this.delayed.size() + ", domainStacks.size() = " + this.domainStacks.size() + ", domainStacksInitSize = " + this.domStackInitSize);
                long loops = sleeptime / 1000;
                long rest = sleeptime % 1000;
                if (loops < 3) {
                  rest = rest + 1000 * loops;
                  loops = 0;
    View Full Code Here
    TOP
    Copyright © 2018 www.massapi.com. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.