Examples of org.jwat.common.ByteCountingPushBackInputStream

org.jwat.common.ByteCountingPushBackInputStream
Basic PushBackInputStream that also keeps track of the number of consumed bytes at any given time. @author nicl


    @Test
    public void test_readerfactoryabstract_isgzip() {
        Assert.assertNotNull(new ReaderFactoryAbstract());
        byte[] bytes;
        ByteCountingPushBackInputStream pbin;
        try {
            ReaderFactoryAbstract.isGzipped(null);
            Assert.fail("Exception expected!");
        } catch (IllegalArgumentException e) {
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Exception not expected!");
        }
        try {
            bytes = GZIP_MAGIC_HEADER;
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), 16);
            Assert.assertTrue(ReaderFactoryAbstract.isGzipped(pbin));
            pbin.close();


            bytes = new byte[] {(byte)0x1f, (byte)0x8b, (byte)0x2f};
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), 16);
            Assert.assertTrue(ReaderFactoryAbstract.isGzipped(pbin));
            pbin.close();


            bytes = new byte[] {(byte)0x8b, (byte)0x1f};
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), 16);
            Assert.assertFalse(ReaderFactoryAbstract.isGzipped(pbin));
            pbin.close();


            bytes = new byte[] {(byte)0x1f};
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), 16);
            Assert.assertFalse(ReaderFactoryAbstract.isGzipped(pbin));
            pbin.close();


            bytes = new byte[] {(byte)0x8b};
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), 16);
            Assert.assertFalse(ReaderFactoryAbstract.isGzipped(pbin));
            pbin.close();
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Exception not expected!");
        }
    }

View Full Code Here


    @Test
    public void test_readerfactoryabstract_isarc() {
        Assert.assertNotNull(new ReaderFactoryAbstract());
        byte[] bytes;
        ByteCountingPushBackInputStream pbin;
        try {
            /*
             * isArcFile().
             */
            bytes = ReaderFactoryAbstract.ARC_MAGIC_HEADER.getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = "filedesc://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = ReaderFactoryAbstract.ARC_MAGIC_HEADER.toUpperCase().getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = "FILEDESC://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = "filedesc".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();
            /*
             * isArcRecord().
             */
            bytes = ReaderFactoryAbstract.ARC_MAGIC_HEADER.getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = "filedesc://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = ReaderFactoryAbstract.ARC_MAGIC_HEADER.toUpperCase().getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = "FILEDESC://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = "filedesc".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Exception not expected!");
        }
        try {
            /*
             * isArcFile().
             */
            bytes = "http://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = "HTTPS://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();


            bytes = "http".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcFile(pbin));
            pbin.close();
            /*
             * isArcRecord().
             */
            bytes = "http://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = "HTTPS://url".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();


            bytes = "http".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isArcRecord(pbin));
            pbin.close();
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Exception not expected!");
        }
    }

View Full Code Here


    @Test
    public void test_readerfactoryabstract_iswarc() {
        Assert.assertNotNull(new ReaderFactoryAbstract());
        byte[] bytes;
        ByteCountingPushBackInputStream pbin;
        try {
            /*
             * isWarcFile.
             */
            bytes = ReaderFactoryAbstract.WARC_MAGIC_HEADER.getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isWarcFile(pbin));
            pbin.close();


            bytes = "WARC/1.0".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isWarcFile(pbin));
            pbin.close();


            bytes = ReaderFactoryAbstract.WARC_MAGIC_HEADER.toLowerCase().getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcFile(pbin));
            pbin.close();


            bytes = "warc/1.0".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcFile(pbin));
            pbin.close();


            bytes = "WARC".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcFile(pbin));
            pbin.close();
            /*
             * isWarcRecord.
             */
            bytes = ReaderFactoryAbstract.WARC_MAGIC_HEADER.getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isWarcRecord(pbin));
            pbin.close();


            bytes = "WARC/1.0".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertTrue(ReaderFactoryAbstract.isWarcRecord(pbin));
            pbin.close();


            bytes = ReaderFactoryAbstract.WARC_MAGIC_HEADER.toLowerCase().getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcRecord(pbin));
            pbin.close();


            bytes = "warc/1.0".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcRecord(pbin));
            pbin.close();


            bytes = "WARC".getBytes();
            pbin = new ByteCountingPushBackInputStream(new ByteArrayInputStream(bytes), PUSHBACK_BUFFER_SIZE);
            Assert.assertFalse(ReaderFactoryAbstract.isWarcRecord(pbin));
            pbin.close();
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail("Exception not expected!");
        }
    }

View Full Code Here


        in = new ByteCountingInputStream( new ByteArrayInputStream( srcArr ) );
        Assert.assertEquals( 1, in.skip( 10 ) );
        Assert.assertEquals( 0, in.skip( 10 ) );


        in = new ByteCountingPushBackInputStream( new ByteArrayInputStream( srcArr ), 1 );
        Assert.assertEquals( 1, in.skip( 10 ) );
        Assert.assertEquals( 0, in.skip( 10 ) );


        sr = new CharCountingStringReader( srcStr );
        Assert.assertEquals( 1, sr.skip( 10 ) );

View Full Code Here

    }


    @Override
    protected boolean readHeader(MaxLengthRecordingInputStream in,
            long payloadLength) throws IOException {
        ByteCountingPushBackInputStream pbin = new ByteCountingPushBackInputStream(in, PUSHBACK_BUFFER_SIZE);
        String versionLine = pbin.readLine();
        String blockDescLine = pbin.readLine();
        // debug
        //System.out.println(versionLine);
        //System.out.println(blockDescLine);
        /*
         * Check for version and parse if present.

View Full Code Here

        }
        currentRecord = null;
        currentReader = reader;
        currentEntry = reader.getNextEntry();
        if (currentEntry != null) {
            ByteCountingPushBackInputStream pbin;
            if (bufferSize > 0) {
                pbin = new ByteCountingPushBackInputStream(
                        new BufferedInputStream(
                                currentEntry.getInputStream(),
                                bufferSize),
                        PUSHBACK_BUFFER_SIZE);
            } else {
                pbin = new ByteCountingPushBackInputStream(
                        currentEntry.getInputStream(), PUSHBACK_BUFFER_SIZE);
            }
            currentRecord = ArcRecordBase.parseRecord(pbin, this);
        }
        if (currentRecord != null) {

View Full Code Here

        }
        currentRecord = null;
        currentReader = new GzipReader(rin);
        currentEntry = currentReader.getNextEntry();
        if (currentEntry != null) {
            ByteCountingPushBackInputStream pbin =
                    new ByteCountingPushBackInputStream(
                            currentEntry.getInputStream(), PUSHBACK_BUFFER_SIZE);
            currentRecord = ArcRecordBase.parseRecord(pbin, this);
        }
        if (currentRecord != null) {
            startOffset = offset;

View Full Code Here

        }
        currentRecord = null;
        currentReader = new GzipReader(rin);
        currentEntry = currentReader.getNextEntry();
        if (currentEntry != null) {
            ByteCountingPushBackInputStream pbin =
                    new ByteCountingPushBackInputStream(
                            new BufferedInputStream(
                                    currentEntry.getInputStream(),
                                    buffer_size),
                            PUSHBACK_BUFFER_SIZE);
            currentRecord = ArcRecordBase.parseRecord(pbin, this);

View Full Code Here

public class TestArcVersionHeader {


    @Test
    public void test_arcversionheader() {
        ByteArrayInputStream in;
        ByteCountingPushBackInputStream pbin;
        ArcFieldParsers fieldParsers = new ArcFieldParsers();
        Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
        fieldParsers.diagnostics = diagnostics;
        ArcVersionHeader header;
        byte[] bytes;
        String digestAlgorithm = null;
        Long length = 0L;
        Object[][] expectedDiagnoses;
        String tmpStr;
        try {
            bytes = new byte[0];
            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);
            /*
             * Invalid parameters.
             */
            try {
                ArcVersionHeader.create(null, null);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                ArcVersionHeader.create(ArcVersion.VERSION_1, null);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                ArcVersionHeader.create(ArcVersion.VERSION_1, "");
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                header = ArcVersionHeader.processPayload(null, length, digestAlgorithm, fieldParsers, diagnostics);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                header = ArcVersionHeader.processPayload(pbin, -1, digestAlgorithm, fieldParsers, diagnostics);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, null, diagnostics);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            try {
                header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, null);
                Assert.fail("Exception expected!");
            } catch (IllegalArgumentException e) {
            }
            /*
             * Empty stream.
             */
            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * Single newline.
             */
            bytes = "\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * 2 newlines.
             */
            bytes = "\n\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * Newline and v1 block desc.
             */
            bytes = ("\n" + ArcConstants.VERSION_1_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(1, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * Newline and v2 block desc.
             */
            bytes = ("\n" + ArcConstants.VERSION_2_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(2, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.
             */
            bytes = "1 0 InternetArchive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.1.
             */
            bytes = "1 1 InternetArchive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1_1, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V2.
             */
            bytes = "2 0 InternetArchive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_2, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * Vx.0.
             */
            bytes = "x 0 InternetArchive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_VERSION_NUMBER + "' value", 2},
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.x.
             */
            bytes = "1 x InternetArchive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertFalse(header.isVersionValid);
            Assert.assertNull(header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_RESERVED + "' value", 2},
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.0 invalid block desc.
             */
            bytes = "1 0 InternetArchive\nAtomic Twister!\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.0 incomplete.
             */
            bytes = "1 0\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.0 too long.
             */
            bytes = "1 0 Internet Archive\n".getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertFalse(header.isValidBlockdDesc);
            Assert.assertEquals(0, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1},
                    {DiagnosisType.ERROR, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V2.0 and v1 block desc.
             */
            bytes = ("2 0 InternetArchive\n" + ArcConstants.VERSION_1_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_2, header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(1, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.0 and v2 block desc.
             */
            bytes = ("1 0 InternetArchive\n" + ArcConstants.VERSION_2_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(2, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * Newline and v2 block desc.
             */
            bytes = ("1 1 InternetArchive\n" + ArcConstants.VERSION_2_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertFalse(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1_1, header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(2, header.blockDescVersion);


            Assert.assertTrue(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 0);


            expectedDiagnoses = new Object[][] {
                    {DiagnosisType.INVALID, ArcConstants.ARC_VERSION_BLOCK, 1}
            };
            TestBaseUtils.compareDiagnoses(expectedDiagnoses, header.diagnostics.getErrors());
            diagnostics.reset();


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.0 and v1 block desc.
             */
            bytes = ("1 0 InternetArchive\n" + ArcConstants.VERSION_1_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertTrue(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1, header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(1, header.blockDescVersion);


            Assert.assertFalse(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 10);


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);
            /*
             * V1.1 and v1 block desc.
             */
            String mdData;
            mdData = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\r\n";
            mdData += "<arcmetadata xmlns:dc=\"http://purl.org/dc/elements/1.1/\"" +
                    " xmlns:dcterms=\"http://purl.org/dc/terms/\"" +
                    " xmlns:arc=\"http://archive.org/arc/1.0/\"" +
                    " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" +
                    " xmlns=\"http://archive.org/arc/1.0/\"" +
                    " xsi:schemaLocation=\"http://archive.org/arc/1.0/ http://www.archive.org/arc/1.0/arc.xsd\">\r\n";
            mdData += "<arc:software>Heritrix @VERSION@ http://crawler.archive.org</arc:software>\r\n";
            mdData += "<arc:hostname>blackbook</arc:hostname>\r\n";
            mdData += "<arc:ip>192.168.1.13</arc:ip>\r\n";
            mdData += "<dcterms:isPartOf>archive.org-shallow</dcterms:isPartOf>\r\n";
            mdData += "<dc:description>archive.org shallow</dc:description>\r\n";
            mdData += "<arc:operator>Admin</arc:operator>\r\n";
            mdData += "<ns0:date xmlns:ns0=\"http://purl.org/dc/elements/1.1/\" xsi:type=\"dcterms:W3CDTF\">2008-04-30T20:48:24+00:00</ns0:date>\r\n";
            mdData += "<arc:http-header-user-agent>Mozilla/5.0 (compatible; heritrix/1.14.0 +http://crawler.archive.org)</arc:http-header-user-agent>\r\n";
            mdData += "<arc:http-header-from>archive-crawler-agent@lists.sourceforge.net</arc:http-header-from>\r\n";
            mdData += "<arc:robots>classic</arc:robots>\r\n";
            mdData += "<dc:format>ARC file version 1.1</dc:format>\r\n";
            mdData += "<dcterms:conformsTo xsi:type=\"dcterms:URI\">http://www.archive.org/web/researcher/ArcFileFormat.php</dcterms:conformsTo>\r\n";
            mdData += "</arcmetadata>\r\n";
            bytes = ("1 1 InternetArchive\n" + ArcConstants.VERSION_1_BLOCK_DEF + "\n" + mdData).getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertTrue(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_1_1, header.version);
            Assert.assertTrue(header.isValidBlockdDesc);
            Assert.assertEquals(1, header.blockDescVersion);


            Assert.assertFalse(header.diagnostics.hasErrors());
            Assert.assertFalse(header.diagnostics.hasWarnings());


            // Save testfile.
            SaveArcTestFiles.saveTestArcVersionHeader(bytes,
                    !header.diagnostics.hasErrors() && !header.diagnostics.hasWarnings(), 11);


            tmpStr = header.toString();
            Assert.assertNotNull(tmpStr);


            /*
             * V2.0 and v2 block desc.
             */
            bytes = ("2 0 InternetArchive\n" + ArcConstants.VERSION_2_BLOCK_DEF + "\n").getBytes();
            length = (long)bytes.length;


            in = new ByteArrayInputStream(bytes);
            pbin = new ByteCountingPushBackInputStream(in, 8192);


            header = ArcVersionHeader.processPayload(pbin, length, digestAlgorithm, fieldParsers, diagnostics);
            Assert.assertTrue(header.isValid());
            Assert.assertTrue(header.isVersionValid);
            Assert.assertEquals(ArcVersion.VERSION_2, header.version);

View Full Code Here

     */
    public GzipReader(InputStream in) {
        if (in == null) {
            throw new IllegalArgumentException("in is null!");
        }
        pbin = new ByteCountingPushBackInputStream(in, DEFAULT_INPUT_BUFFER_SIZE);
        inputBytes = new byte[DEFAULT_INPUT_BUFFER_SIZE];
    }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.jwat.common.ByteCountingPushBackInputStream

org.archive.wayback.resourcestore.jwat.JWATResource

org.jwat.arc.ArcReaderCompressed

org.jwat.arc.ArcReaderFactory

org.jwat.arc.ArcReaderUncompressed

org.jwat.arc.ArcVersionHeader

org.jwat.arc.TestArcHeader

org.jwat.arc.TestArcReaderFactory_IsMagic

org.jwat.arc.TestArcReaderUncompressed

org.jwat.arc.TestArcVersionHeader

org.jwat.archive.common.TestReaderFactoryAbstract

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.