Examples of net.yacy.cora.protocol.RequestHeader

net.yacy.cora.protocol.RequestHeader


        // determine the file date
        final Date fileDate = ftpClient.entryDate(path);
        
        // create response header
        RequestHeader requestHeader = new RequestHeader();
        if (request.referrerhash() != null) {
            DigestURI refurl = sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash());
            if (refurl != null) requestHeader.put(RequestHeader.REFERER, refurl.toNormalform(true, false));
        }
        ResponseHeader responseHeader = new ResponseHeader();
        responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(fileDate));
        responseHeader.put(HeaderFramework.CONTENT_TYPE, mime);

View Full Code Here

            path = path.substring(0, argpos);
        }


        // set up virtual connection properties to call httpdFileHander.doGet()
        final HashMap<String, Object> conProp = new HashMap<String, Object>();
        final RequestHeader header = new RequestHeader(HTTPDemon.reverseMappingCache);
        conProp.put(HeaderFramework.CONNECTION_PROP_METHOD, HeaderFramework.METHOD_GET);
        conProp.put(HeaderFramework.CONNECTION_PROP_PATH, path);
        conProp.put(HeaderFramework.CONNECTION_PROP_ARGS, args);
        conProp.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
        conProp.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, requesthost);
        header.put(RequestHeader.AUTHORIZATION, authorization);
        if (requestHeader.containsKey(RequestHeader.COOKIE))
          header.put(RequestHeader.COOKIE, requestHeader.get(RequestHeader.COOKIE));
        header.put(RequestHeader.REFERER, requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH));
        HTTPDFileHandler.doGet(conProp, header, out);
    }

View Full Code Here

            // parsing the http request line
            final HashMap<String, Object> prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);


            // we now know the HTTP version. depending on that, we read the header
            String httpVersion = (String) prop.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_0_9;
            final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
                    ? new RequestHeader(reverseMappingCache)
                    : readHeader(prop, session);


            // handling transparent proxy support
            handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);

View Full Code Here

        try {
            final HashMap<String, Object> prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session);


            // we now know the HTTP version. depending on that, we read the header
            String httpVersion = (String) prop.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_0_9;
            final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
                    ? new RequestHeader(reverseMappingCache)
                    : readHeader(prop,session);


            // handle transparent proxy support
            handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);

View Full Code Here

        try {
            final HashMap<String, Object> prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session);


            // we now know the HTTP version. depending on that, we read the header
            String httpVersion = (String) prop.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_0_9;
            final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
                    ? new RequestHeader(reverseMappingCache)
                    : readHeader(prop, session);


            // handle transfer-coding
            final InputStream sessionIn;
            final String transferEncoding = header.get(HeaderFramework.TRANSFER_ENCODING);
            if (transferEncoding != null) {
                if (!HeaderFramework.HTTP_VERSION_1_1.equals(httpVersion)) {
                    log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
                }
                if("chunked".equalsIgnoreCase(header.get(HeaderFramework.TRANSFER_ENCODING))) {
                    sessionIn = new ChunkedInputStream(session.in);
                } else {
                    // "A server which receives an entity-body with a transfer-coding it does
                    // not understand SHOULD return 501 (Unimplemented), and close the
                    // connection." [RFC 2616, section 3.6]

View Full Code Here

        prop.put(HeaderFramework.CONNECTION_PROP_PATH, "/");
        prop.put(HeaderFramework.CONNECTION_PROP_EXT, "");
        prop.put(HeaderFramework.CONNECTION_PROP_URL, "");


        // parse remaining lines
        final RequestHeader header = readHeader(prop,session);


        if (!(allowProxy(session))) {
            // not authorized through firewall blocking (ip does not match filter)
            session.out.write(UTF8.getBytes(httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + session.userAddress.getHostAddress() + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING));
            return serverCore.TERMINATE_CONNECTION;

View Full Code Here




    public static RequestHeader readHeader(final HashMap<String, Object> prop, final serverCore.Session theSession) throws IOException {


        // reading all headers
        final RequestHeader header = new RequestHeader(HTTPDemon.reverseMappingCache);
        int p;
        String line;
        while ((line = theSession.readLineAsString()) != null) {
            if (line.length() == 0) break; // this separates the header of the HTTP request from the body
            // parse the header line: a property separated with the ':' sign
            if ((p = line.indexOf(':')) >= 0) {
                // store a property
                header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
            }
        }


        /*
         * doing some header validation here ...
         */
        String httpVersion = (String) prop.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); if (httpVersion == null) httpVersion = "HTTP/0.9";
        if (httpVersion.equals("HTTP/1.1") && !header.containsKey(HeaderFramework.HOST)) {
            // the HTTP/1.1 specification requires that an HTTP/1.1 server must reject any
            // HTTP/1.1 message that does not contain a Host header.
            HTTPDemon.sendRespondError(prop,theSession.out,0,400,null,null,null);
            throw new IOException("400 Bad request");
        }

View Full Code Here

            if (cachedResponse != null && content != null) {
                // yes we have the content


                // create request header values and a response object because we need that
                // in case that we want to return the cached content in the next step
                final RequestHeader requestHeader = new RequestHeader();
                requestHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());
                DigestURI refererURL = null;
                if (request.referrerhash() != null) refererURL = this.sb.getURL(Segments.Process.LOCALCRAWLING, request.referrerhash());
                if (refererURL != null) requestHeader.put(RequestHeader.REFERER, refererURL.toNormalform(true, true));
                final Response response = new Response(
                        request,
                        requestHeader,
                        cachedResponse,
                        "200",

View Full Code Here

        return da;
    }
    
    public static SitemapReader parse(final DigestURI sitemapURL) throws IOException {
        // download document
        final RequestHeader requestHeader = new RequestHeader();
        requestHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());
        final HTTPClient client = new HTTPClient();
        client.setTimout(5000);
        client.setHeader(requestHeader.entrySet());
        try {
            client.GET(sitemapURL.toString());
            if (client.getStatusCode() != 200) {
                throw new IOException("Unable to download the sitemap file " + sitemapURL +
                        "\nServer returned status: " + client.getHttpResponse().getStatusLine());

View Full Code Here

        // read password
        String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
        if (encodedPassword == null) encodedPassword = ""; // not defined


        // send 'wget' to web interface
        final RequestHeader requestHeader = new RequestHeader();
        requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
//        final Client con = new Client(10000, requestHeader);
        final HTTPClient con = new HTTPClient();
        con.setHeader(requestHeader.entrySet());
//        ResponseContainer res = null;
        try {
//            res = con.GET("http://localhost:"+ port +"/" + path);
            con.GETbytes("http://localhost:"+ port +"/" + path);

View Full Code Here

0 1 2 3 4

TOP

Related Classes of net.yacy.cora.protocol.RequestHeader

de.anomic.crawler.retrieval.FileLoader

de.anomic.crawler.retrieval.FTPLoader

de.anomic.crawler.retrieval.HTTPLoader

de.anomic.crawler.retrieval.Response

de.anomic.crawler.retrieval.SMBLoader

de.anomic.crawler.RobotsTxt

de.anomic.http.server.HTTPDemon

de.anomic.http.server.ServerSideIncludes

de.anomic.search.Switchboard

de.anomic.server.serverSwitch

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.