Package org.archive.io.arc

Examples of org.archive.io.arc.ARCRecord

151
152
153
154
155
156
157
158
159
160
161
            public String getVersion() {
                return null;
            }

        };
        ARCRecord r = new ARCRecord(new ByteArrayInputStream(hdr.getBytes()),
                arh, 0, false, true, false);

        HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true);
        har.skipHttpHeader();
        byte[] b = new byte[BODY.length()];
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
        final String url = "http://www.archive.org/index.htm";
        final String arcHeader = url
                + " 192.168.0.1 20070515111004 text/html 167568\n";
        final String hdr = arcHeader + HTTPHEADER + BODY;

        ARCRecord r = new ARCRecord(new ByteArrayInputStream(hdr.getBytes()),
                "READER_IDENTIFIER", 0, false, true, false);

        HeaderedArchiveRecord har = new HeaderedArchiveRecord(r, true);
        har.skipHttpHeader();
        byte[] b = new byte[BODY.length()];
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    /* (non-Javadoc)
     * @see org.archive.wayback.util.Adapter#adapt(java.lang.Object)
     */
    public ARCRecord adapt(ArchiveRecord o) {
      ARCRecord rec = null;
      if(o instanceof ARCRecord) {
        rec = (ARCRecord) o;
      }
      return rec;
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
    public void testPlainHttpRecord() throws Exception {
        String payload = "hogehogehogehogehoge";
        WARCRecordInfo recinfo = TestWARCRecordInfo.createHttpResponse(payload);
        recinfo.setMimetype("text/plain");
        TestARCReader ar = new TestARCReader(recinfo);
        ARCRecord rec = ar.get(0);
        ArcResource res = new ArcResource(rec, ar);
        res.parseHeaders();
       
        assertEquals("statusCode", 200, res.getStatusCode());
        assertEquals("content-type", "text/plain", res.getHeader("Content-Type"));
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
        WARCRecordInfo recinfo = new TestWARCRecordInfo(
                TestWARCRecordInfo.buildHttpResponseBlock("200 OK",
                        "text/plain", payload.getBytes("UTF-8"), true));
        recinfo.setMimetype("text/plain");
        TestARCReader ar = new TestARCReader(recinfo);
        ARCRecord rec = ar.get(0);
        ArcResource res = new ArcResource(rec, ar);
        res.parseHeaders();
       
        assertEquals("statusCode", 200, res.getStatusCode());
        assertEquals("content-type", "text/plain", res.getHeader("Content-Type"));
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
        WARCRecordInfo recinfo = new TestWARCRecordInfo(
                TestWARCRecordInfo.buildCompressedHttpResponseBlock(ctype,
                        payload.getBytes()));
        recinfo.setMimetype(ctype);
        TestARCReader ar = new TestARCReader(recinfo);
        ARCRecord rec = ar.get(0);
        ArcResource res = new ArcResource(rec, ar);
        res.parseHeaders();
       
        assertEquals("statusCode", 200, res.getStatusCode());
        assertEquals("content-type", ctype, res.getHeader("Content-Type"));
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
        WARCRecordInfo recinfo = new TestWARCRecordInfo(
                TestWARCRecordInfo.buildCompressedHttpResponseBlock(ctype,
                        payload.getBytes(), true));
        recinfo.setMimetype(ctype);
        TestARCReader ar = new TestARCReader(recinfo);
        ARCRecord rec = ar.get(0);
        ArcResource res = new ArcResource(rec, ar);
        res.parseHeaders();
       
        assertEquals("statusCode", 200, res.getStatusCode());
        assertEquals("content-type", ctype, res.getHeader("Content-Type"));
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
      try {
        int status = http.executeMethod(method);
        if(status == 200) {

          ByteArrayInputStream bais = new ByteArrayInputStream(method.getResponseBody());
          ARCRecord r = new ARCRecord(
              new GZIPInputStream(bais),
              "id",0L,false,false,true);
          ArcResource ar = (ArcResource)
            ResourceFactory.ARCArchiveRecordToResource(r, null);
          if(ar.getStatusCode() == 502) {
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
          HttpClient http = new HttpClient(connectionManager);
          http.setHostConfiguration(hostConfiguration);

          int status = http.executeMethod(method);
          if(status == 200) {
            ARCRecord r =
              new ARCRecord(new GZIPInputStream(
                  method.getResponseBodyAsStream()),
                  "id",0L,false,false,true);
            Resource res = null;
            try {
          res = ResourceFactory.ARCArchiveRecordToResource(r, null);
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
      r = (ArcResource) cache.getCachedResource(url, maxCacheMS , false);
    } finally {
      PerfStats.timeEnd(PerfStat.LiveWeb);
    }

    ARCRecord ar = (ARCRecord) r.getArcRecord();
    int status = ar.getStatusCode();
    if ((status == 200) || ((status >= 300) && (status < 400))) {
      result.setCaptureTimestamp(ar.getMetaData().getDate());
      result.setMimeType(ar.getMetaData().getMimetype());
      return r;
    }
   
    return null;
  }
View Full Code Here
TOP

Related Classes of org.archive.io.arc.ARCRecord

  • org.archive.access.nutch.jobs.ImportArcs
  • org.archive.io.Arc2Warc
  • org.archive.io.ArchiveTest
  • org.archive.io.HeaderedArchiveRecordTest
  • org.archive.mapred.ARCMapRunner$IndexingThread
  • org.archive.wayback.liveweb.ARCUnwrappingProxy
  • org.archive.wayback.liveweb.LiveWebCache
  • org.archive.wayback.liveweb.RemoteLiveWebCache
  • org.archive.wayback.resourceindex.indexer.hadoop.Driver$MapClass
  • org.archive.wayback.resourcestore.ArcIndexer$ArchiveRecordToARCRecordAdapter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.