Package net.yacy.cora.protocol.http

Examples of net.yacy.cora.protocol.http.HTTPClient$IdledConnectionEvictor


      params = new DefaultSolrParams( this._invariantParams, params );
    }


    byte[] result = null;
    final HTTPClient client = new HTTPClient();
    if (this.solraccount.length() > 0 && this.solrpw.length() > 0 && this.host.length() > 0) {
        HTTPClient.setAuth(this.host, this.port, this.solraccount, this.solrpw);
    }

    if (SolrRequest.METHOD.POST == request.getMethod()) {
        final boolean isMultipart = ( streams != null && streams.size() > 1 );
        if (streams == null || isMultipart) {
            String url = this._baseURL + path;

            final HashMap<String, ContentBody> parts = new HashMap<String, ContentBody>();
            final Iterator<String> iter = params.getParameterNamesIterator();
            while (iter.hasNext()) {
                final String p = iter.next();
                final String[] vals = params.getParams(p);
                if (vals != null) {
                    for (final String v : vals) {
                        if (isMultipart) {
                            parts.put(p, new StringBody(v, utf8));
                          } else {
                              if (url.indexOf('?') >= 0) url += "&" + p + "=" + v; else url += "?" + p + "=" + v;
                          }
                    }
                }
            }

            if (isMultipart) {
                for (final ContentStream content : streams) {
                  parts.put(content.getName(), new InputStreamBody(content.getStream(), content.getContentType(), null));
                }
            }

            try {
                result = client.POSTbytes(url, parts, true);
            } finally {
                client.finish();
            }
        } else {
            // It has one stream, this is the post body, put the params in the URL
            final String pstr = ClientUtils.toQueryString(params, false);
            final String url = this._baseURL + path + pstr;

            // Single stream as body
            // Using a loop just to get the first one
            final ContentStream[] contentStream = new ContentStream[1];
            for (final ContentStream content : streams) {
              contentStream[0] = content;
              break;
            }
            result = client.POSTbytes(url, contentStream[0].getStream(), contentStream[0].getStream().available());
        }
    } else if (SolrRequest.METHOD.GET == request.getMethod()) {
        result = client.GETbytes( this._baseURL + path + ClientUtils.toQueryString( params, false ));
    } else {
        throw new SolrServerException("Unsupported method: "+request.getMethod() );
    }

      final int statusCode = client.getStatusCode();
      if (statusCode != 200) {
        throw new IOException("bad status code: " + statusCode + ", " + client.getHttpResponse().getStatusLine() + ", url = " + this._baseURL + path);
      }

      // Read the contents
      //System.out.println("SOLR RESPONSE: " + UTF8.String(result));
      final InputStream respBody = new ByteArrayInputStream(result);
View Full Code Here


    }
    private static byte[] postToFile(final SeedDB seedDB, final String targetHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
      return postToFile(seedDB.targetAddress(targetHash), targetHash, filename, parts, timeout);
    }
    private static byte[] postToFile(final String targetAddress, final String targetPeerHash, final String filename, final Map<String,ContentBody> parts, final int timeout) throws IOException {
        final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
        return httpClient.POSTbytes(new MultiProtocolURI("http://" + targetAddress + "/yacy/" + filename), Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh", parts, false);
    }
View Full Code Here

            parts.put("magic", UTF8.StringBody(Long.toString(Network.magic)));
            parts.put("seed", UTF8.StringBody(mySeed.genSeedStr(salt)));
            // send request
            final long start = System.currentTimeMillis();
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), 30000, yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", parts);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 30000);
            final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/hello.html"), Seed.b64Hash2hexHash(otherHash) + ".yacyh", parts, false);
            Network.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + address + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + (System.currentTimeMillis() - start) + " milliseconds");
            result = FileUtils.table(content);
        } catch (final Exception e) {
            if (Thread.currentThread().isInterrupted()) {
                Network.log.logInfo("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted.");
View Full Code Here

            final Map<String,ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
            parts.put("call", UTF8.StringBody("remotecrawl"));
            parts.put("count", UTF8.StringBody(Integer.toString(maxCount)));
            parts.put("time", UTF8.StringBody(Long.toString(maxTime)));
            // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), (int) maxTime);
            final byte[] result = httpClient.POSTbytes(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false);
            final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
            if (reader == null) {
                Network.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer (1), reader == null");
                target.put(Seed.RCOUNT, "0");
                seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
View Full Code Here

            parts.put("reason", UTF8.StringBody(reason));
            parts.put("wordh", UTF8.StringBody(wordhashes));
            parts.put("lurlEntry", UTF8.StringBody(((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt))));
            // send request
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), 10000, target.getHexHash() + ".yacyh", parts);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 10000);
            final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), target.getHexHash() + ".yacyh", parts, false);
            return FileUtils.table(content);
        } catch (final Exception e) {
            // most probably a network time-out exception
            Network.log.logWarning("yacyClient.crawlReceipt error:" + e.getMessage());
            return null;
View Full Code Here

            final Map<String,ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
            parts.put("wordc", UTF8.StringBody(Integer.toString(indexes.size())));
            parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));
            parts.put("indexes", UTF8.StringBody(entrypost.toString()));
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
            final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
            final Iterator<String> v = FileUtils.strings(content);
            // this should return a list of urlhashes that are unknown

            final Map<String, String> result = FileUtils.table(v);
            // return the transfered index data in bytes (for debugging only)
View Full Code Here

            }
        }
        try {
            parts.put("urlc", UTF8.StringBody(Integer.toString(urlc)));
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
            final byte[] content = httpClient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
            final Iterator<String> v = FileUtils.strings(content);

            final Map<String, String> result = FileUtils.table(v);
            // return the transfered url data in bytes (for debugging only)
            result.put("urlPayloadSize", Integer.toString(urlPayloadSize));
View Full Code Here

        String address = targetSeed.getClusterAddress();
        if (address == null) { address = "localhost:8090"; }
        try {
            final Map<String,ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetSeed.hash, salt);
            // final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), 5000, targetSeed.getHexHash() + ".yacyh", parts);
            final HTTPClient httpclient = new HTTPClient(ClientIdentification.getUserAgent(), 5000);
            final byte[] content = httpclient.POSTbytes(new MultiProtocolURI("http://" + address + "/yacy/profile.html"), targetSeed.getHexHash() + ".yacyh", parts, false);
            return FileUtils.table(content);
        } catch (final Exception e) {
            Network.log.logWarning("yacyClient.getProfile error:" + e.getMessage());
            return null;
        }
View Full Code Here

            newpost.put("process", UTF8.StringBody("permission"));
            newpost.put("purpose", UTF8.StringBody("crcon"));
      byte[] res;
      try {
        // res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true);
        final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
        res = httpClient.POSTbytes(url, vhost, newpost, true);
        System.out.println(UTF8.String(res));
      } catch (final IOException e1) {
        Log.logException(e1);
      }
        }
View Full Code Here

            parts.put("constraint", UTF8.StringBody((constraint == null) ? "" : constraint.exportB64()));
            if (secondarySearchSuperviser != null) parts.put("abstracts", UTF8.StringBody("auto"));
            // resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
            //resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.crawlerUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/search.html"), 60000, target.getHexHash() + ".yacyh", parts));

            final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), 60000);
            resultMap = FileUtils.table(httpClient.POSTbytes(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false));

            // evaluate request result
            if (resultMap == null || resultMap.isEmpty()) throw new IOException("resultMap is NULL");
            try {
                this.searchtime = Integer.parseInt(resultMap.get("searchtime"));
View Full Code Here

TOP

Related Classes of net.yacy.cora.protocol.http.HTTPClient$IdledConnectionEvictor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.