Examples of org.jwat.common.Payload

org.jwat.common.Payload
This class makes the archived payload of an ARC/WARC record accessible through a stream. The stream is fixed length so only the payload is available. @author lbihanic, selghissassi, nicl

                }


                writer.writeHeader(record);


                if ( record.hasPayload() ) {
                    Payload payload = record.getPayload();
                    //writer.transfer( payload.getInputStream(), payload.getTotalLength() );
                    writer.streamPayload( payload.getInputStreamComplete() );
                }


                writer.closeRecord();
            }

View Full Code Here

                }


                writer.writeRawHeader(record.header.headerBytes, record.header.contentLength);


                if ( record.hasPayload() ) {
                    Payload payload = record.getPayload();
                    //writer.transfer( payload.getInputStream(), payload.getTotalLength() );
                    writer.streamPayload( payload.getInputStreamComplete() );
                }


                writer.closeRecord();
            }

View Full Code Here


        byte[] srcArr = new byte[ 0 ];
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        byte[] dstArr;


        Payload payload;


        try {
            payload = Payload.processPayload( null, 0, 16, null );
            Assert.fail( "Exception expected!" );
        } catch (IOException e) {
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
        }


        try {
            payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), -1, 16, null );
            Assert.fail( "Exception expected!" );
        } catch (IOException e) {
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
        }


        try {
            payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), 0, -1, null );
            Assert.fail( "Exception expected!" );
        } catch (IOException e) {
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
        }


        try {
            payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), 0, 0, null );
            Assert.fail( "Exception expected!" );
        } catch (IOException e) {
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
        }


        try {
            payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), 0, 16, "shit1" );
            Assert.assertNull( payload.getDigest() );
        } catch (IOException e) {
            e.printStackTrace();
        }


        InputStream in;
        long remaining;
        byte[] tmpBuf = new byte[ 256 ];
        int read;


        MessageDigest md = null;
        try {
            md = MessageDigest.getInstance( "SHA1" );
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }


        for ( int r=0; r<runs; ++r) {
            closed = 0;
            for ( int n=min; n<max; ++n ) {
                srcArr = new byte[ n ];
                random.nextBytes( srcArr );


                out.reset();


                try {
                    /*
                     * Payload.
                     */
                    payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), srcArr.length, 16, digestAlgorithm );
                    Assert.assertNull(payload.onClosedHandler);
                    payload.setOnClosedHandler( this );
                    Assert.assertEquals(this, payload.onClosedHandler);
                    payload.setOnClosedHandler( null );
                    Assert.assertNull(payload.onClosedHandler);
                    payload.setOnClosedHandler( this );
                    Assert.assertEquals(this, payload.onClosedHandler);


                    Assert.assertNull( payload.getPayloadHeaderWrapped() );
                    payload.setPayloadHeaderWrapped( null );
                    Assert.assertNull( payload.getPayloadHeaderWrapped() );


                    in = payload.getInputStream();
                    Assert.assertEquals( payload.getInputStreamComplete(), payload.getInputStream() );
                    Assert.assertEquals( srcArr.length, payload.getTotalLength() );
                    Assert.assertEquals( 16, payload.getPushbackSize() );


                    remaining = payload.getTotalLength();
                    read = 0;
                    while ( remaining > 0 && read != -1 ) {
                        out.write(tmpBuf, 0, read);
                        remaining -= read;


                        // This wont work with buffered streams...
                        //Assert.assertEquals( remaining, payload.getUnavailable() );
                        //Assert.assertEquals( remaining, payload.getRemaining() );


                        read = random.nextInt( 15 ) + 1;
                        read = in.read(tmpBuf, 0, read);
                    }
                    Assert.assertEquals( 0, remaining );
                    Assert.assertEquals( 0, payload.getUnavailable() );
                    Assert.assertEquals( 0, payload.getRemaining() );


                    dstArr = out.toByteArray();
                    Assert.assertEquals( srcArr.length, dstArr.length );
                    Assert.assertArrayEquals( srcArr, dstArr );


                    Assert.assertFalse(payload.isClosed());
                    in.close();
                    Assert.assertFalse(payload.isClosed());
                    payload.close();
                    Assert.assertEquals( n, closed );
                    Assert.assertTrue(payload.isClosed());


                    in.close();
                    payload.close();


                    // Only possible in test, because of access to protected fields.
                    payload.bClosed = false;
                    payload.close();
                    /*
                     * Digest.
                     */
                    if ( digestAlgorithm != null ) {
                        Assert.assertNotNull( payload.getDigest() );


                        byte[] digest1 = payload.getDigest();


                        md.reset();
                        byte[] digest2 = md.digest( srcArr );


                        Assert.assertArrayEquals( digest1, digest2 );
                    } else {
                        Assert.assertNull( payload.getDigest() );
                    }
                } catch (IOException e) {
                    Assert.fail( "Exception not expected!" );
                    e.printStackTrace();
                }

View Full Code Here


        byte[] srcArr = new byte[ 0 ];
        //ByteArrayOutputStream out = new ByteArrayOutputStream();
        //byte[] dstArr;


        Payload payload;


        //InputStream in;
        //long remaining;
        //byte[] tmpBuf = new byte[ 256 ];
        //int read;


        MessageDigest md = null;
        try {
            md = MessageDigest.getInstance( "SHA1" );
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }


        for ( int r=0; r<runs; ++r) {
            closed = 0;
            for ( int n=min; n<max; ++n ) {
                srcArr = new byte[ n ];
                random.nextBytes( srcArr );


                //out.reset();


                try {
                    /*
                     * Payload.
                     */
                    payload = Payload.processPayload( new ByteArrayInputStream( srcArr ), srcArr.length, 16, digestAlgorithm );
                    Assert.assertNull(payload.onClosedHandler);
                    payload.setOnClosedHandler( this );
                    Assert.assertEquals(this, payload.onClosedHandler);
                    payload.setOnClosedHandler( null );
                    Assert.assertNull(payload.onClosedHandler);
                    payload.setOnClosedHandler( this );
                    Assert.assertEquals(this, payload.onClosedHandler);


                    Assert.assertNull( payload.getPayloadHeaderWrapped() );
                    payload.setPayloadHeaderWrapped( null );
                    Assert.assertNull( payload.getPayloadHeaderWrapped() );


                    //in = payload.getInputStream();
                    Assert.assertEquals( payload.getInputStreamComplete(), payload.getInputStream() );
                    Assert.assertEquals( srcArr.length, payload.getTotalLength() );
                    Assert.assertEquals( 16, payload.getPushbackSize() );


                    /*
                    remaining = payload.getTotalLength();
                    read = 0;
                    while ( remaining > 0 && read != -1 ) {
                        out.write(tmpBuf, 0, read);
                        remaining -= read;


                        // This wont work with buffered streams...
                        //Assert.assertEquals( remaining, payload.getUnavailable() );
                        //Assert.assertEquals( remaining, payload.getRemaining() );


                        read = random.nextInt( 15 ) + 1;
                        read = in.read(tmpBuf, 0, read);
                    }
                    Assert.assertEquals( 0, remaining );
                    */


                    Assert.assertFalse(payload.isClosed());
                    //in.close();
                    Assert.assertFalse(payload.isClosed());
                    payload.close();
                    Assert.assertEquals( n, closed );
                    Assert.assertTrue(payload.isClosed());


                    Assert.assertEquals( 0, payload.getUnavailable() );
                    // Stream closed exception.
                    //Assert.assertEquals( 0, payload.getRemaining() );


                    //dstArr = out.toByteArray();
                    //Assert.assertEquals( srcArr.length, dstArr.length );
                    //Assert.assertArrayEquals( srcArr, dstArr );


                    //in.close();
                    payload.close();


                    // Only possible in test, because of access to protected fields.
                    payload.bClosed = false;
                    payload.close();
                    /*
                     * Digest.
                     */
                    if ( digestAlgorithm != null ) {
                        Assert.assertNotNull( payload.getDigest() );


                        byte[] digest1 = payload.getDigest();


                        md.reset();
                        byte[] digest2 = md.digest( srcArr );


                        Assert.assertArrayEquals( digest1, digest2 );
                    } else {
                        Assert.assertNull( payload.getDigest() );
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                    Assert.fail( "Exception not expected!" );
                }

View Full Code Here

        String in_file;
        InputStream in;
        ByteCountingInputStream bcin;
        ArcReader reader;
        ArcRecordBase record;
        Payload payload;
        PayloadWithHeaderAbstract payloadWithHeader;
        InputStream in_payload;
        byte[] tmpBuf = new byte[8192];
        int read;
        Object[][] expectedDiagnoses;
        int records;
        String tmpStr;


        MessageDigest md_block = null;
        if (blockAlgo != null) {
            try {
                md_block = MessageDigest.getInstance(blockAlgo);
            } catch (NoSuchAlgorithmException e) {
                e.printStackTrace();
            }
        }


        MessageDigest md_payload = null;
        if (payloadAlgo != null) {
            try {
                md_payload = MessageDigest.getInstance(payloadAlgo);
            } catch (NoSuchAlgorithmException e) {
                e.printStackTrace();
            }
        }


        byte[] block_digest;
        byte[] payload_digest;


        try {
            /*
             * Uncompressed.
             */
            in_file = "IAH-20080430204825-00000-blackbook.arc";


            in = this.getClass().getClassLoader().getResourceAsStream(in_file);
            bcin = new ByteCountingInputStream(in);
            reader = ArcReaderFactory.getReader(bcin, 8192);
            reader.setBlockDigestAlgorithm(blockAlgo);
            reader.setBlockDigestEncoding(blockBase);
            reader.setBlockDigestEnabled(true);
            reader.setPayloadDigestAlgorithm(payloadAlgo);
            reader.setPayloadDigestEncoding(payloadBase);
            reader.setPayloadDigestEnabled(true);


            records = 0;
            while ((record = reader.getNextRecord()) != null) {
                payload = record.getPayload();
                Assert.assertEquals(record.payload, payload);
                payloadWithHeader = null;
                if (payload != null) {
                    Assert.assertTrue(record.hasPayload());
                    payloadWithHeader = payload.getPayloadHeaderWrapped();
                } else {
                    Assert.assertFalse(record.hasPayload());
                }


                if (md_payload != null) {
                    md_payload.reset();
                }
                if (md_block != null && payload != null) {
                    md_block.reset();
                }


                block_digest = null;
                payload_digest = null;


                if (md_payload != null && payloadWithHeader != null) {
                    md_block.update(payloadWithHeader.getHeader());


                    //System.out.println("--");
                    //System.out.println(new String(payloadWithHeader.getHeader()));
                    //System.out.println("--");


                    in_payload = payloadWithHeader.getPayloadInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                        md_payload.update(tmpBuf, 0, read);
                    }
                } else if (md_block != null && payload != null) {
                    in_payload = payload.getInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                    }
                }


                Assert.assertFalse(record.isClosed());
                record.close();
                Assert.assertTrue(record.isClosed());
                ++records;


                if (md_block != null && payload != null) {
                    block_digest = md_block.digest();
                    //System.out.println("b1: " + Base16.encodeArray(block_digest));
                    //System.out.println("b2: " + Base16.encodeArray(payload.getDigest()));
                    Assert.assertArrayEquals(block_digest, payload.getDigest());


                    Assert.assertEquals(blockAlgo, record.computedBlockDigest.algorithm);
                    if (!"base2".equals(blockBase)) {
                        Assert.assertEquals(blockBase, record.computedBlockDigest.encoding);
                    } else {
                        Assert.assertNull(record.computedBlockDigest.encoding);
                    }
                    Assert.assertArrayEquals(record.computedBlockDigest.digestBytes, payload.getDigest());
                } else {
                    Assert.assertNull(record.computedBlockDigest);
                }
                if (md_payload != null && payloadWithHeader != null) {
                    payload_digest = md_payload.digest();
                    //System.out.println("p1: " + Base16.encodeArray(payload_digest));
                    //System.out.println("p2: " + Base16.encodeArray(payloadWithHeader.getDigest()));
                    Assert.assertArrayEquals(payload_digest, payloadWithHeader.getDigest());


                    Assert.assertEquals(payloadAlgo, record.computedPayloadDigest.algorithm);
                    if (!"base2".equals(payloadBase)) {
                        Assert.assertEquals(payloadBase, record.computedPayloadDigest.encoding);
                    } else {
                        Assert.assertNull(record.computedPayloadDigest.encoding);
                    }
                    Assert.assertArrayEquals(record.computedPayloadDigest.digestBytes, payloadWithHeader.getDigest());
                } else {
                    Assert.assertNull(record.computedPayloadDigest);
                }


                if (!"base2".equals(blockBase) && !"base2".equals(payloadBase)) {
                    Assert.assertTrue(record.isCompliant());
                    Assert.assertFalse(record.diagnostics.hasErrors());
                    Assert.assertFalse(record.diagnostics.hasWarnings());
                } else {
                    Assert.assertFalse(record.isCompliant());
                    if (payloadWithHeader != null) {
                        expectedDiagnoses = new Object[][] {
                                {DiagnosisType.INVALID_DATA, "Block digest encoding scheme", 1},
                                {DiagnosisType.INVALID_DATA, "Payload digest encoding scheme", 1}
                        };
                    } else if (payload != null) {
                        expectedDiagnoses = new Object[][] {
                                {DiagnosisType.INVALID_DATA, "Block digest encoding scheme", 1},
                        };
                    } else {
                        expectedDiagnoses = new Object[0][0];
                    }
                    TestBaseUtils.compareDiagnoses(expectedDiagnoses, record.diagnostics.getErrors());
                    Assert.assertFalse(record.diagnostics.hasWarnings());
                }


                tmpStr = record.toString();
                Assert.assertNotNull(tmpStr);
            }
            Assert.assertEquals(300, records);
            Assert.assertEquals(bcin.getConsumed(), reader.getConsumed());
            Assert.assertEquals(bcin.getConsumed(), reader.getOffset());


            reader.close();


            Assert.assertEquals(300, records);
            Assert.assertEquals(bcin.getConsumed(), reader.getConsumed());
            Assert.assertEquals(bcin.getConsumed(), reader.getOffset());
            /*
             * Compressed.
             */
            in_file = "IAH-20080430204825-00000-blackbook.arc.gz";


            in = this.getClass().getClassLoader().getResourceAsStream(in_file);
            bcin = new ByteCountingInputStream(in);
            reader = ArcReaderFactory.getReader(bcin, 8192);
            reader.setBlockDigestAlgorithm(blockAlgo);
            reader.setBlockDigestEncoding(blockBase);
            reader.setBlockDigestEnabled(true);
            reader.setPayloadDigestAlgorithm(payloadAlgo);
            reader.setPayloadDigestEncoding(payloadBase);
            reader.setPayloadDigestEnabled(true);


            records = 0;
            while ((record = reader.getNextRecord()) != null) {
                payload = record.getPayload();
                Assert.assertEquals(record.payload, payload);
                payloadWithHeader = null;
                if (payload != null) {
                    Assert.assertTrue(record.hasPayload());
                    payloadWithHeader = payload.getPayloadHeaderWrapped();
                } else {
                    Assert.assertFalse(record.hasPayload());
                }


                if (md_payload != null) {
                    md_payload.reset();
                }
                if (md_block != null && payload != null) {
                    md_block.reset();
                }


                block_digest = null;
                payload_digest = null;


                if (md_payload != null && payloadWithHeader != null) {
                    md_block.update(payloadWithHeader.getHeader());


                    //System.out.println("--");
                    //System.out.println(new String(payloadWithHeader.getHeader()));
                    //System.out.println("--");


                    in_payload = payloadWithHeader.getPayloadInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                        md_payload.update(tmpBuf, 0, read);
                    }
                } else if (md_block != null && payload != null) {
                    in_payload = payload.getInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                    }
                }


                Assert.assertFalse(record.isClosed());
                record.close();
                Assert.assertTrue(record.isClosed());
                ++records;


                if (md_block != null && payload != null) {
                    block_digest = md_block.digest();
                    //System.out.println("b1: " + Base16.encodeArray(block_digest));
                    //System.out.println("b2: " + Base16.encodeArray(payload.getDigest()));
                    Assert.assertArrayEquals(block_digest, payload.getDigest());


                    Assert.assertEquals(blockAlgo, record.computedBlockDigest.algorithm);
                    if (!"base2".equals(blockBase)) {
                        Assert.assertEquals(blockBase, record.computedBlockDigest.encoding);
                    } else {
                        Assert.assertNull(record.computedBlockDigest.encoding);
                    }
                    Assert.assertArrayEquals(record.computedBlockDigest.digestBytes, payload.getDigest());
                } else {
                    Assert.assertNull(record.computedBlockDigest);
                }
                if (md_payload != null && payloadWithHeader != null) {
                    payload_digest = md_payload.digest();
                    //System.out.println("p1: " + Base16.encodeArray(payload_digest));
                    //System.out.println("p2: " + Base16.encodeArray(payloadWithHeader.getDigest()));
                    Assert.assertArrayEquals(payload_digest, payloadWithHeader.getDigest());


                    Assert.assertEquals(payloadAlgo, record.computedPayloadDigest.algorithm);
                    if (!"base2".equals(payloadBase)) {
                        Assert.assertEquals(payloadBase, record.computedPayloadDigest.encoding);
                    } else {
                        Assert.assertNull(record.computedPayloadDigest.encoding);
                    }
                    Assert.assertArrayEquals(record.computedPayloadDigest.digestBytes, payloadWithHeader.getDigest());
                } else {
                    Assert.assertNull(record.computedPayloadDigest);
                }


                if (!"base2".equals(blockBase) && !"base2".equals(payloadBase)) {
                    Assert.assertTrue(record.isCompliant());
                    Assert.assertFalse(record.diagnostics.hasErrors());
                    Assert.assertFalse(record.diagnostics.hasWarnings());
                } else {
                    Assert.assertFalse(record.isCompliant());
                    if (payloadWithHeader != null) {
                        expectedDiagnoses = new Object[][] {
                                {DiagnosisType.INVALID_DATA, "Block digest encoding scheme", 1},
                                {DiagnosisType.INVALID_DATA, "Payload digest encoding scheme", 1}
                        };
                    } else if (payload != null) {
                        expectedDiagnoses = new Object[][] {
                                {DiagnosisType.INVALID_DATA, "Block digest encoding scheme", 1},
                        };
                    } else {
                        expectedDiagnoses = new Object[0][0];
                    }
                    TestBaseUtils.compareDiagnoses(expectedDiagnoses, record.diagnostics.getErrors());
                    Assert.assertFalse(record.diagnostics.hasWarnings());
                }


                tmpStr = record.toString();
                Assert.assertNotNull(tmpStr);
            }
            Assert.assertEquals(300, records);
            Assert.assertEquals(bcin.getConsumed(), reader.getConsumed());
            Assert.assertEquals(bcin.getConsumed(), reader.getOffset());


            reader.close();


            Assert.assertEquals(300, records);
            Assert.assertEquals(bcin.getConsumed(), reader.getConsumed());
            Assert.assertEquals(bcin.getConsumed(), reader.getOffset());
            /*
             * Small file.
             */
            in_file = "small_BNF.arc";


            in = this.getClass().getClassLoader().getResourceAsStream(in_file);
            bcin = new ByteCountingInputStream(in);
            reader = ArcReaderFactory.getReader(bcin, 8192);
            reader.setBlockDigestAlgorithm(blockAlgo);
            reader.setBlockDigestEncoding(blockBase);
            reader.setBlockDigestEnabled(true);
            reader.setPayloadDigestAlgorithm(payloadAlgo);
            reader.setPayloadDigestEncoding(payloadBase);
            reader.setPayloadDigestEnabled(true);


            records = 0;
            while ((record = reader.getNextRecord()) != null) {
                payload = record.getPayload();
                Assert.assertEquals(record.payload, payload);
                payloadWithHeader = null;
                if (payload != null) {
                    Assert.assertTrue(record.hasPayload());
                    payloadWithHeader = payload.getPayloadHeaderWrapped();
                } else {
                    Assert.assertFalse(record.hasPayload());
                }


                if (md_payload != null) {
                    md_payload.reset();
                }
                if (md_block != null && payload != null) {
                    md_block.reset();
                }


                block_digest = null;
                payload_digest = null;


                if (md_payload != null && payloadWithHeader != null) {
                    md_block.update(payloadWithHeader.getHeader());


                    //System.out.println("--");
                    //System.out.println(new String(payloadWithHeader.getHeader()));
                    //System.out.println("--");


                    in_payload = payloadWithHeader.getPayloadInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                        md_payload.update(tmpBuf, 0, read);
                    }
                } else if (md_block != null && payload != null) {
                    in_payload = payload.getInputStream();
                    while ((read = in_payload.read(tmpBuf)) != -1) {
                        md_block.update(tmpBuf, 0, read);
                    }
                }


                Assert.assertFalse(record.isClosed());
                record.close();
                Assert.assertTrue(record.isClosed());
                ++records;


                if (md_block != null && payload != null) {
                    block_digest = md_block.digest();
                    //System.out.println("b1: " + Base16.encodeArray(block_digest));
                    //System.out.println("b2: " + Base16.encodeArray(payload.getDigest()));
                    Assert.assertArrayEquals(block_digest, payload.getDigest());


                    Assert.assertEquals(blockAlgo, record.computedBlockDigest.algorithm);
                    if (!"base2".equals(blockBase)) {
                        Assert.assertEquals(blockBase, record.computedBlockDigest.encoding);
                    } else {
                        Assert.assertNull(record.computedBlockDigest.encoding);
                    }
                    Assert.assertArrayEquals(record.computedBlockDigest.digestBytes, payload.getDigest());
                } else {
                    Assert.assertNull(record.computedBlockDigest);
                }
                if (md_payload != null && payloadWithHeader != null) {
                    payload_digest = md_payload.digest();

View Full Code Here

                if (bDebugOutput) {
                    if (record.header.warcFilename != null) {
                        saveUtf8(record.header.warcFilename);
                    }


                    Payload payload = record.getPayload();
                    if (payload != null) {
                        InputStream pis = payload.getInputStream();
                        long l = payload.getTotalLength();
                        System.out.println("Payload length: " + l);
                        savePayload(pis);
                    }
                }

View Full Code Here

    @Test
    public void test_arcreaderfactory_autodetect() {
        boolean bDebugOutput = System.getProperty("jwat.debug.output") != null;


        InputStream in;
        Payload payload;
        InputStream payload_in;


        ArcReader reader;
        ArcRecordBase record;

View Full Code Here

      }
    }
    else {
      in = r.pbin;
    }
    Payload payload = null;
    HttpHeader httpHeader = null;
    if (ArcReaderFactory.isArcRecord(in)) {
      r.arcReader = ArcReaderFactory.getReaderUncompressed();
      r.arcReader.setUriProfile(UriProfile.RFC3986_ABS_16BIT_LAX);
      r.arcReader.setBlockDigestEnabled(false);
      r.arcReader.setPayloadDigestEnabled(false);
      r.arcRecord = r.arcReader.getNextRecordFrom(in, offset);
      if (r.arcRecord != null) {
        payload = r.arcRecord.getPayload();
        if (payload != null) {
          httpHeader = r.arcRecord.getHttpHeader();
        }
        if (httpHeader != null) {
          r.payloadStream = httpHeader.getPayloadInputStream();
          r.length = httpHeader.payloadLength;
          r.status = httpHeader.statusCode;
        } else if (payload != null) {
          r.payloadStream = payload.getInputStreamComplete();
          r.length = payload.getTotalLength();
          r.status = 200;
        } else {
          r.payloadStream = new ByteArrayInputStream(new byte[0]);
          r.length = 0;
          r.status = 200;
        }
      }
    }
    else if ( WarcReaderFactory.isWarcRecord(in) ) {
      r.warcReader = WarcReaderFactory.getReaderUncompressed();
      r.warcReader.setWarcTargetUriProfile(UriProfile.RFC3986_ABS_16BIT_LAX);
      r.warcReader.setBlockDigestEnabled(false);
      r.warcReader.setPayloadDigestEnabled(false);
      r.warcRecord = r.warcReader.getNextRecordFrom(in, offset);
      if (r.warcRecord != null) {
        payload = r.warcRecord.getPayload();
        if (payload != null) {
          httpHeader = r.warcRecord.getHttpHeader();
        }
        if (httpHeader != null) {
          r.payloadStream = httpHeader.getPayloadInputStream();
          r.length = httpHeader.payloadLength;
          r.status = httpHeader.statusCode;
        } else if (payload != null) {
          r.payloadStream = payload.getInputStreamComplete();
          r.length = payload.getTotalLength();
          r.status = 200;
        } else {
          r.payloadStream = new ByteArrayInputStream(new byte[0]);
          r.length = 0;
          r.status = 200;

View Full Code Here

TOP

Related Classes of org.jwat.common.Payload

org.archive.wayback.resourcestore.jwat.JWATResource

org.jwat.arc.TestArcReader_Digest

org.jwat.arc.TestArcReaderFactory

org.jwat.common.TestPayload

org.jwat.warc.TestWarc_Utf8

org.jwat.warc.TestWarcWriterFactory

java.io.BufferedInputStream

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.