Package bixo.datum

Examples of bixo.datum.HttpHeaders

117
118
119
120
121
122
123
124
125
126
127
        } catch (InterruptedException e) {
            // Break out of our delay, but preserve interrupt state.
            Thread.currentThread().interrupt();
        }

        HttpHeaders headers = new HttpHeaders();
        headers.add("x-responserate", "" + bytesPerSecond);
        FetchedDatum result = new FetchedDatum(url, url, System.currentTimeMillis(), headers, new ContentBytes(new byte[contentSize]), "text/html", bytesPerSecond);
        result.setPayload(payload);
        return result;
    }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
 
  @Test
  public void testSplitterWithNonMbox() {
    MboxSplitterFunction splitter = new MboxSplitterFunction();
   
    FetchedDatum datum = new FetchedDatum("baseUrl", "redirectedUrl", 0, new HttpHeaders(), new ContentBytes(), "text/ascii", 0);
    TupleEntry value = new TupleEntry(datum.getTupleEntry());
   
    when(_funcCall.getArguments()).thenReturn(value);
    splitter.operate(_process, _funcCall);
   
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
  public void testSplitterTwoEmails() throws UnsupportedEncodingException {
    MboxSplitterFunction splitter = new MboxSplitterFunction();

    final String mboxString = "From 1\r\rContent 1\r\rFrom 2\r\rContent 2";
    byte[] mboxContent = mboxString.getBytes("us-ascii");
    FetchedDatum datum = new FetchedDatum("baseUrl", "redirectedUrl", 0, new HttpHeaders(), new ContentBytes(mboxContent), "application/mbox", 0);
    TupleEntry value = new TupleEntry(datum.getTupleEntry());
   
    when(_funcCall.getArguments()).thenReturn(value);
    splitter.operate(_process, _funcCall);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
    String html = readFromFile("parser-files/base-url.html");
   
    // Create FetchedDatum using data
    String url = "http://olddomain.com/base-url.html";
    String contentType = "text/html; charset=utf-8";
    HttpHeaders headers = new HttpHeaders();
    headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
    ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
    FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
   
    // Call parser.parse
    SimpleParser parser = new SimpleParser();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
    // Create FetchedDatum using data
    String url = "http://olddomain.com/relative-urls.html";
    String location = "http://newdomain.com";
   
    String contentType = "text/html; charset=utf-8";
    HttpHeaders headers = new HttpHeaders();
    headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
    headers.add(HttpHeaderNames.CONTENT_LOCATION, location);
    ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
    FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
   
    // Call parser.parse
    SimpleParser parser = new SimpleParser();
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
    // Create FetchedDatum using data
    String url = "http://olddomain.com/relative-urls.html";
    String location = "redirected/";
   
    String contentType = "text/html; charset=utf-8";
    HttpHeaders headers = new HttpHeaders();
    headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
    headers.add(HttpHeaderNames.CONTENT_LOCATION, location);
    ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
    FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
   
    // Call parser.parse
    SimpleParser parser = new SimpleParser();
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
    // Create FetchedDatum using data
    String url = "http://olddomain.com/relative-urls.html";
    String redirectedUrl = "http://newdomain.com";
   
    String contentType = "text/html; charset=utf-8";
    HttpHeaders headers = new HttpHeaders();
    headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
    ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
    FetchedDatum fetchedDatum = new FetchedDatum(url, redirectedUrl, System.currentTimeMillis(), headers, content, contentType, 0);
   
    // Call parser.parse
    SimpleParser parser = new SimpleParser();
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
189
       
        // Create FetchedDatum using data
        String url = "http://domain.com/all-link-types.html";
       
        String contentType = "text/html; charset=utf-8";
        HttpHeaders headers = new HttpHeaders();
        headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
        ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
        FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
       
        // Call parser.parse
        SimpleParser parser = new SimpleParser();
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
217
       
        // Create FetchedDatum using data
        String url = "http://domain.com/all-link-types.html";
       
        String contentType = "text/html; charset=utf-8";
        HttpHeaders headers = new HttpHeaders();
        headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
        ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
        FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
       
        // Call parser.parse
        ParserPolicy policy = new ParserPolicy( ParserPolicy.DEFAULT_MAX_PARSE_DURATION,
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
256
       
        // Create FetchedDatum using data
        String url = "http://domain.com/all-link-types.html";
       
        String contentType = "text/html; charset=utf-8";
        HttpHeaders headers = new HttpHeaders();
        headers.add(HttpHeaderNames.CONTENT_TYPE, contentType);
        ContentBytes content = new ContentBytes(html.getBytes("utf-8"));
        FetchedDatum fetchedDatum = new FetchedDatum(url, url, System.currentTimeMillis(), headers, content, contentType, 0);
       
        // Call parser.parse
        Set<String> linkTags =
View Full Code Here
TOP

Related Classes of bixo.datum.HttpHeaders

  • bixo.exceptions.HttpFetchExceptionTest
  • bixo.fetcher.LoggingFetcher
  • bixo.fetcher.SimpleHttpFetcher
  • bixo.fetcher.simulation.FakeHttpFetcher
  • bixo.parser.SimpleParserTest
  • bixo.pipes.AbstractFetchPipeTest
  • bixo.pipes.AbstractFetchPipeTest$CustomFetcher
  • bixo.pipes.ParsePipeTest
  • cascading.tuple.Tuple
  • com.scaleunlimited.helpful.operations.MboxSplitterFunctionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.