Package org.commoncrawl.io.shared

Examples of org.commoncrawl.io.shared.NIOHttpHeaders.findValue()

168
169
170
171
172
173
174
175
176
177
178
      // retured bytes represent the header(encoded in utf-8), terminated by a \r\n\r\n. The content follows this terminator
      // we search for this specific byte pattern to locate start of content, then compare it against source ...
      Assert.assertTrue(ArcFileReaderTests.compareTo(testRecord.data,0,testRecord.data.length,value.getContent().getReadOnlyBytes(),value.getContent().getOffset(),value.getContent().getCount()) == 0);
      NIOHttpHeaders headers = ArcFileItemUtils.buildHeaderFromArcFileItemHeaders(value.getHeaderItems());
      // validate metadata
      Assert.assertEquals("text/html",headers.findValue(Constants.ARCFileHeader_ARC_MimeType));
      Assert.assertEquals(value.getArcFilePos(),testRecord.streamPos);
      Assert.assertEquals(value.getArcFileSize(),testRecord.rawSize);
      Assert.assertEquals("test-value", headers.findValue("test"));
      Assert.assertEquals(value.getArcFileName(),((FileSplit)split).getPath().getName());
     
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
      NIOHttpHeaders headers = ArcFileItemUtils.buildHeaderFromArcFileItemHeaders(value.getHeaderItems());
      // validate metadata
      Assert.assertEquals("text/html",headers.findValue(Constants.ARCFileHeader_ARC_MimeType));
      Assert.assertEquals(value.getArcFilePos(),testRecord.streamPos);
      Assert.assertEquals(value.getArcFileSize(),testRecord.rawSize);
      Assert.assertEquals("test-value", headers.findValue("test"));
      Assert.assertEquals(value.getArcFileName(),((FileSplit)split).getPath().getName());
     
    }
    reader.close();
   
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
      // retured bytes represent the header(encoded in utf-8), terminated by a \r\n\r\n. The content follows this terminator
      // we search for this specific byte pattern to locate start of content, then compare it against source ...
      Assert.assertTrue(ArcFileReaderTests.compareTo(testRecord.data,0,testRecord.data.length,value.getContent().getReadOnlyBytes(),value.getContent().getOffset(),value.getContent().getCount()) == 0);
      NIOHttpHeaders headers = ArcFileItemUtils.buildHeaderFromArcFileItemHeaders(value.getHeaderItems());
      // validate metadata
      Assert.assertEquals("text/html",headers.findValue(Constants.ARCFileHeader_ARC_MimeType));
      Assert.assertEquals(value.getArcFilePos(),testRecord.streamPos);
      Assert.assertEquals(value.getArcFileSize(),testRecord.rawSize);
      Assert.assertEquals("test-value", headers.findValue("test"));
      Assert.assertEquals(value.getArcFileName(),((FileSplit)split).getPath().getName());
     
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
      NIOHttpHeaders headers = ArcFileItemUtils.buildHeaderFromArcFileItemHeaders(value.getHeaderItems());
      // validate metadata
      Assert.assertEquals("text/html",headers.findValue(Constants.ARCFileHeader_ARC_MimeType));
      Assert.assertEquals(value.getArcFilePos(),testRecord.streamPos);
      Assert.assertEquals(value.getArcFileSize(),testRecord.rawSize);
      Assert.assertEquals("test-value", headers.findValue("test"));
      Assert.assertEquals(value.getArcFileName(),((FileSplit)split).getPath().getName());
     
    }
    reader.close();
   
View Full Code Here
471
472
473
474
475
476
477
478
479
480
481
        // read headers ...
        String headersText = new String(value.getBytes(),0,indexofHeaderTerminator,Charset.forName("UTF-8"));
        NIOHttpHeaders headers = NIOHttpHeaders.parseHttpHeaders(headersText);
        for (int i=0;i<testRecord.headers.size();++i) {
          Pair<String,String> testHeaderRecord = testRecord.headers.get(i);
          Assert.assertNotNull(headers.findValue(testHeaderRecord.e0));
          Assert.assertEquals(testHeaderRecord.e1,headers.findValue(testHeaderRecord.e0));
        }
       
        Assert.assertTrue(compareTo(testRecord.data,0,testRecord.data.length,value.getBytes(),indexofHeaderTerminator,testRecord.data.length) == 0);
      }
View Full Code Here
472
473
474
475
476
477
478
479
480
481
482
        String headersText = new String(value.getBytes(),0,indexofHeaderTerminator,Charset.forName("UTF-8"));
        NIOHttpHeaders headers = NIOHttpHeaders.parseHttpHeaders(headersText);
        for (int i=0;i<testRecord.headers.size();++i) {
          Pair<String,String> testHeaderRecord = testRecord.headers.get(i);
          Assert.assertNotNull(headers.findValue(testHeaderRecord.e0));
          Assert.assertEquals(testHeaderRecord.e1,headers.findValue(testHeaderRecord.e0));
        }
       
        Assert.assertTrue(compareTo(testRecord.data,0,testRecord.data.length,value.getBytes(),indexofHeaderTerminator,testRecord.data.length) == 0);
      }
      reader.close();
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    String headerStr = new TextBytes(rawArcPayload.getBytes(),0,headerLen,true).toString();
    NIOHttpHeaders headers = NIOHttpHeaders.parseHttpHeaders(headerStr);
   
    // extract appropriate header values to populate ArcFileItem
    arcFileItem.getUriAsTextBytes().set(key, true);
    arcFileItem.setHostIP(headers.findValue(Constants.ARCFileHeader_HostIP));
    try {
      arcFileItem.setTimestamp(TIMESTAMP14.parse(headers.findValue(Constants.ARCFileHeader_ARC_Timestamp)).getTime());
    } catch (Exception e) {
      LOG.error("Invalid Timestamp Encountered in Item Metdata. URL:"
          + arcFileItem.getUri() + " Timestamp:" + headers.findValue(Constants.ARCFileHeader_ARC_Timestamp) );
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
   
    // extract appropriate header values to populate ArcFileItem
    arcFileItem.getUriAsTextBytes().set(key, true);
    arcFileItem.setHostIP(headers.findValue(Constants.ARCFileHeader_HostIP));
    try {
      arcFileItem.setTimestamp(TIMESTAMP14.parse(headers.findValue(Constants.ARCFileHeader_ARC_Timestamp)).getTime());
    } catch (Exception e) {
      LOG.error("Invalid Timestamp Encountered in Item Metdata. URL:"
          + arcFileItem.getUri() + " Timestamp:" + headers.findValue(Constants.ARCFileHeader_ARC_Timestamp) );
    }
    arcFileItem.setMimeType(headers.findValue(Constants.ARCFileHeader_ARC_MimeType));
View Full Code Here
410
411
412
413
414
415
416
417
418
419
420
  public void HttpConnectionStateChanged(NIOHttpConnection theConnection,State oldState, State state) {
   
    if (oldState == State.PARSING_HEADERS && state == State.RECEIVING_CONTENT) {
      NIOHttpHeaders headers = theConnection.getResponseHeaders();
      LOG.info("*** S3 INCOMING HEADERS:" + headers.toString());
      LOG.info("Content Length From Header for:" + theConnection.getURL() + " is:" + headers.findValue("Content-Length"));
    }
   
    LOG.info("S3Download Connection:" + theConnection.getURL() +" Old State:" + oldState + " NewState:" + state);
    // get context
    S3DownloadItem item = (S3DownloadItem) theConnection.getContext();
View Full Code Here
435
436
437
438
439
440
441
442
443
444
445
        // if success
        if (resultCode >= 200 && resultCode <300) {
         
          continueDownloading = true;
         
          String etagValue = headers.findValue("ETag");
          String contentLengthValue = headers.findValue("Content-Length");
          String rangeValue = headers.findValue("Content-Range");
         
          if(etagValue != null && contentLengthValue != null) {
            try {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.