final LockableRandomAccessBufferFactory smallRAFFactory = new ByteArrayRandomAccessBufferFactory();

    final FilenameGenerator fg;

    final PersistentFileTracker persistentFileTracker;

    final LockableRandomAccessBufferFactory bigRAFFactory;

    final BucketFactory smallBucketFactory;

    final BucketFactory bigBucketFactory;

    final File dir;

    final InsertContext baseContext;

    final WaitableExecutor executor;

    final Ticker ticker;

    final byte cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256;

    final byte[] cryptoKey;

    final ChecksumChecker checker;

    final MemoryLimitedJobRunner memoryLimitedJobRunner;

    final PersistentJobRunner jobRunner;

    final KeySalter salt = new KeySalter() {

        @Override

        public byte[] saltKey(Key key) {

            return key.getRoutingKey();

        }

       

    };

    private final FreenetURI URI;

   

    public SplitFileInserterStorageTest() throws IOException {

        dir = new File("split-file-inserter-storage-test");

        dir.mkdir();

        executor = new WaitableExecutor(new PooledExecutor());

        ticker = new CheatingTicker(executor);

        RandomSource r = new DummyRandomSource(12345);

        fg = new FilenameGenerator(r, true, dir, "freenet-test");

        persistentFileTracker = new TrivialPersistentFileTracker(dir, fg);

        bigRAFFactory = new PooledFileRandomAccessBufferFactory(fg, r);

        smallBucketFactory = new ArrayBucketFactory();

        bigBucketFactory = new TempBucketFactory(executor, fg, 0, 0, r, false, 0, null);

        baseContext = HighLevelSimpleClientImpl.makeDefaultInsertContext(bigBucketFactory, new SimpleEventProducer());

        cryptoKey = new byte[32];

        r.nextBytes(cryptoKey);

        checker = new CRCChecksumChecker();

        memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 20, executor);

        jobRunner = new DummyJobRunner(executor, null);

        URI = FreenetURI.generateRandomCHK(r);

    }

   

    class MyCallback implements SplitFileInserterStorageCallback {

       

        private boolean finishedEncode;

        private boolean hasKeys;

        private boolean succeededInsert;

        private InsertException failed;

        private Metadata metadata;

        @Override

        public synchronized void onFinishedEncode() {

            finishedEncode = true;

            notifyAll();

        }

        @Override

        public synchronized void onHasKeys() {

            hasKeys = true;

            notifyAll();

        }

       

        @Override

        public void encodingProgress() {

            // Ignore.

        }

       

        private void checkFailed() throws InsertException {

            if(failed != null)

                throw failed;

        }

        public synchronized void waitForFinishedEncode() throws InsertException {

            while(!finishedEncode) {

                checkFailed();

                try {

                    wait();

                } catch (InterruptedException e) {

                    // Ignore.

                }

            }

        }

        public synchronized void waitForHasKeys() throws InsertException {

            while(!hasKeys) {

                checkFailed();

                try {

                    wait();

                } catch (InterruptedException e) {

                    // Ignore.

                }

            }

        }

        @Override

        public synchronized void onSucceeded(Metadata metadata) {

            succeededInsert = true;

            this.metadata = metadata;

            notifyAll();

        }

       

        public synchronized Metadata waitForSucceededInsert() throws InsertException {

            while(!succeededInsert) {

                checkFailed();

                try {

                    wait();

                } catch (InterruptedException e) {

                    // Ignore.

                }

            }

            return metadata;

        }

        @Override

        public synchronized void onFailed(InsertException e) {

            failed = e;

            notifyAll();

        }

        @Override

        public void onInsertedBlock() {

            // Ignore.

        }

        @Override

        public void clearCooldown() {

            // Ignore.

        }

        public synchronized boolean hasFailed() {

            return failed != null;

        }

        public synchronized boolean hasFinishedEncode() {

            return finishedEncode;

        }

    }

   

    public void testSmallSplitfileNoLastBlock() throws IOException, InsertException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(),

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        assertEquals(storage.segments[0].dataBlockCount, 2);

        assertEquals(storage.segments[0].checkBlockCount, 3);

        assertEquals(storage.segments[0].crossCheckBlockCount, 0);

        assertTrue(storage.getStatus() == Status.ENCODED);

    }

    public void testSmallSplitfileWithLastBlock() throws IOException, InsertException {

        Random r = new Random(12122);

        long size = 65535;

        byte[] originalData = new byte[(int)size];

        r.nextBytes(originalData);

        LockableRandomAccessBuffer data = smallRAFFactory.makeRAF(originalData, 0, originalData.length, true);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(),

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        // Now check the data blocks...

        assertEquals(storage.segments.length, 1);

        assertEquals(storage.segments[0].dataBlockCount, 2);

        assertEquals(storage.segments[0].checkBlockCount, 3);

        assertEquals(storage.segments[0].crossCheckBlockCount, 0);

        assertTrue(Arrays.equals(storage.readSegmentDataBlock(0, 0), Arrays.copyOfRange(originalData, 0, CHKBlock.DATA_LENGTH)));

        int truncateLength = (int) (size % CHKBlock.DATA_LENGTH);

        long offsetLastBlock = size - truncateLength;

        byte[] buf = storage.readSegmentDataBlock(0, 1);

        assert(buf.length == CHKBlock.DATA_LENGTH);

        byte[] truncated = Arrays.copyOfRange(buf, 0, truncateLength);

        byte[] originalLastBlock = Arrays.copyOfRange(originalData, (int)offsetLastBlock, originalData.length);

        assertEquals(originalLastBlock.length, truncated.length);

        assertTrue(Arrays.equals(originalLastBlock, truncated));

        assertTrue(storage.getStatus() == Status.ENCODED);

    }

   

    public void testSmallSplitfileHasKeys() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.earlyEncode = true;

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        assertEquals(storage.segments[0].dataBlockCount, 2);

        assertEquals(storage.segments[0].checkBlockCount, 3);

        assertEquals(storage.segments[0].crossCheckBlockCount, 0);

        cb.waitForHasKeys();

        for(int i=0;i<storage.segments[0].dataBlockCount+storage.segments[0].checkBlockCount+storage.segments[0].crossCheckBlockCount;i++)

            storage.segments[0].readKey(i);

        storage.encodeMetadata();

        assertTrue(storage.getStatus() == Status.ENCODED);

    }

    public void testSmallSplitfileCompletion() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        for(int i=0;i<segment.totalBlockCount;i++) {

            segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey());

        }

        cb.waitForSucceededInsert();

        assertEquals(storage.getStatus(), Status.SUCCEEDED);

    }

    public void testSmallSplitfileChooseCompletion() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.maxInsertRetries = 2;

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        boolean[] chosenBlocks = new boolean[segment.totalBlockCount];

        // Choose and fail all blocks.

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            assertTrue(chosen != null);

            keys.addInsert(chosen);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.onFailure(chosen.blockNumber, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        }

        keys.clear();

        // Choose and succeed all blocks.

        chosenBlocks = new boolean[segment.totalBlockCount];

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            keys.addInsert(chosen);

            assertTrue(chosen != null);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());

        }

        cb.waitForSucceededInsert();

        assertEquals(storage.getStatus(), Status.SUCCEEDED);

    }

    public void testSmallSplitfileChooseCooldown() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.maxInsertRetries = 2;

        context.consecutiveRNFsCountAsSuccess = 2;

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        boolean[] chosenBlocks = new boolean[segment.totalBlockCount];

        assertFalse(storage.noBlocksToSend());

        // Choose and fail all blocks.

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            assertTrue(chosen != null);

            keys.addInsert(chosen);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

        }

        assertNull(storage.chooseBlock());

        assertTrue(storage.noBlocksToSend());

        for(int i=0;i<segment.totalBlockCount;i++) {

            segment.onFailure(i, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

            assertFalse(storage.noBlocksToSend());

        }

        keys.clear();

        // Choose and succeed all blocks.

        chosenBlocks = new boolean[segment.totalBlockCount];

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            keys.addInsert(chosen);

            assertTrue(chosen != null);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());

        }

        cb.waitForSucceededInsert();

        assertEquals(storage.getStatus(), Status.SUCCEEDED);

    }

    public void testSmallSplitfileChooseCooldownNotRNF() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.maxInsertRetries = 2;

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        boolean[] chosenBlocks = new boolean[segment.totalBlockCount];

        assertFalse(storage.noBlocksToSend());

        // Choose and fail all blocks.

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            assertTrue(chosen != null);

            keys.addInsert(chosen);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

        }

        assertNull(storage.chooseBlock());

        assertTrue(storage.noBlocksToSend());

        for(int i=0;i<segment.totalBlockCount;i++) {

            // We need to test this path too.

            segment.onFailure(i, new InsertException(InsertExceptionMode.REJECTED_OVERLOAD));

            assertFalse(storage.noBlocksToSend());

        }

        keys.clear();

        // Choose and succeed all blocks.

        chosenBlocks = new boolean[segment.totalBlockCount];

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            keys.addInsert(chosen);

            assertTrue(chosen != null);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());

        }

        cb.waitForSucceededInsert();

        assertEquals(storage.getStatus(), Status.SUCCEEDED);

    }

    public void testSmallSplitfileConsecutiveRNFsHack() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.maxInsertRetries = 0;

        context.consecutiveRNFsCountAsSuccess = 2;

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        boolean[] chosenBlocks = new boolean[segment.totalBlockCount];

        // First RNF.

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            assertTrue(chosen != null);

            keys.addInsert(chosen);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.setKey(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());

            segment.onFailure(chosen.blockNumber, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        }

        chosenBlocks = new boolean[segment.totalBlockCount];

        // Second RNF.

        keys.clear();

        for(int i=0;i<segment.totalBlockCount;i++) {

            BlockInsert chosen = segment.chooseBlock();

            assertTrue(chosen != null);

            keys.addInsert(chosen);

            assertFalse(chosenBlocks[chosen.blockNumber]);

            chosenBlocks[chosen.blockNumber] = true;

            segment.onFailure(chosen.blockNumber, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        }

        // Should count as success at this point.

        cb.waitForSucceededInsert();

        assertEquals(storage.getStatus(), Status.SUCCEEDED);

    }

    public void testSmallSplitfileConsecutiveRNFsHackFailure() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        // Do 2 RNFs and then a RejectedOverload. Should fail at that point.

        context.maxInsertRetries = 2;

        context.consecutiveRNFsCountAsSuccess = 3;

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        segment.setKey(0, segment.encodeBlock(0).getClientKey());

        segment.onFailure(0, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        assertEquals(storage.getStatus(), Status.ENCODED);

        segment.onFailure(0, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        assertEquals(storage.getStatus(), Status.ENCODED);

        segment.onFailure(0, new InsertException(InsertExceptionMode.REJECTED_OVERLOAD));

        // Should count as success at this point.

        try {

            cb.waitForSucceededInsert();

            assertTrue(false);

        } catch (InsertException e) {

            // Expected.

            assertEquals(e.mode, InsertExceptionMode.TOO_MANY_RETRIES_IN_BLOCKS);

            assertTrue(e.errorCodes != null);

            assertEquals(e.errorCodes.getErrorCount(InsertExceptionMode.ROUTE_NOT_FOUND), 2);

            assertEquals(e.errorCodes.getErrorCount(InsertExceptionMode.REJECTED_OVERLOAD), 1);

            assertEquals(e.errorCodes.totalCount(), 3);

            assertEquals(storage.getStatus(), Status.FAILED);

        }

    }

    public void testSmallSplitfileFailureMaxRetries() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.consecutiveRNFsCountAsSuccess = 0;

        context.maxInsertRetries = 2;

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        for(int i=0;i<3;i++) {

            segment.onFailure(0, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));

        }

        try {

            cb.waitForSucceededInsert();

            assertTrue(false);

        } catch (InsertException e) {

            assertEquals(e.mode, InsertExceptionMode.TOO_MANY_RETRIES_IN_BLOCKS);

            assertTrue(e.errorCodes != null);

            assertEquals(e.errorCodes.getErrorCount(InsertExceptionMode.ROUTE_NOT_FOUND), 3);

            assertEquals(e.errorCodes.totalCount(), 3);

            assertEquals(storage.getStatus(), Status.FAILED);

        }

    }

    public void testSmallSplitfileFailureFatalError() throws IOException, InsertException, MissingKeyException {

        Random r = new Random(12121);

        long size = 65536; // Exact multiple, so no last block

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.maxInsertRetries = 2;

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertEquals(storage.segments.length, 1);

        SplitFileInserterSegmentStorage segment = storage.segments[0];

        assertEquals(segment.dataBlockCount, 2);

        assertEquals(segment.checkBlockCount, 3);

        assertEquals(segment.crossCheckBlockCount, 0);

        assertEquals(storage.getStatus(), Status.ENCODED);

        assertTrue(InsertException.isFatal(InsertExceptionMode.INTERNAL_ERROR));

        segment.onFailure(0, new InsertException(InsertExceptionMode.INTERNAL_ERROR));

        try {

            cb.waitForSucceededInsert();

            assertTrue(false);

        } catch (InsertException e) {

            assertEquals(e.mode, InsertExceptionMode.FATAL_ERRORS_IN_BLOCKS);

            assertTrue(e.errorCodes != null);

            assertEquals(e.errorCodes.getErrorCount(InsertExceptionMode.INTERNAL_ERROR), 1);

            assertEquals(e.errorCodes.totalCount(), 1);

            assertEquals(storage.getStatus(), Status.FAILED);

        }

    }

    private HashResult[] getHashes(LockableRandomAccessBuffer data) throws IOException {

        InputStream is = new RAFInputStream(data, 0, data.size());

        MultiHashInputStream hashStream = new MultiHashInputStream(is, HashType.SHA256.bitmask);

        FileUtil.copy(is, new NullOutputStream(), data.size());

        is.close();

        return hashStream.getResults();

    }

    private LockableRandomAccessBuffer generateData(Random random, long size,

            LockableRandomAccessBufferFactory smallRAFFactory) throws IOException {

        LockableRandomAccessBuffer thing = smallRAFFactory.makeRAF(size);

        BucketTools.fill(thing, random, 0, size);

        return new ReadOnlyRandomAccessBuffer(thing);

    }

   

    public void testRoundTripSimple() throws FetchException, MetadataParseException, Exception {

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*2, CompatibilityMode.COMPAT_CURRENT);

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*2-1, CompatibilityMode.COMPAT_CURRENT);

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*128, CompatibilityMode.COMPAT_CURRENT);

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*128+1, CompatibilityMode.COMPAT_CURRENT);

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*192, CompatibilityMode.COMPAT_CURRENT);

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*192+1, CompatibilityMode.COMPAT_CURRENT);

    }

   

    public void testRoundTripOneBlockSegment() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception {

        testRoundTripSimpleRandom(CHKBlock.DATA_LENGTH*(128+1)-1, CompatibilityMode.COMPAT_1250_EXACT);

    }

   

    public void testRoundTripCrossSegment() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception {

        if(!TestProperty.EXTENSIVE) return;

        // Test cross-segment:

        testRoundTripCrossSegmentRandom(CHKBlock.DATA_LENGTH*128*21);

    }

   

    public void testRoundTripDataBlocksOnly() throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception {

        testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH*128*5);

        if(!TestProperty.EXTENSIVE) return;

        // Test cross-segment:

        testRoundTripCrossSegmentDataBlocks(CHKBlock.DATA_LENGTH*128*21);

    }

   

    public void testResumeCrossSegment() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        if(!TestProperty.EXTENSIVE) return;

        testResumeCrossSegment(CHKBlock.DATA_LENGTH*128*21);

    }

   

    public void testEncodeAfterShutdownCrossSegment() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        if(!TestProperty.EXTENSIVE) return;

        testEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*21);

    }

   

    public void testRepeatedEncodeAfterShutdown() throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*5); // Not cross-segment.

        if(!TestProperty.EXTENSIVE) return;

        testRepeatedEncodeAfterShutdownCrossSegment(CHKBlock.DATA_LENGTH*128*21); // Cross-segment.

    }

   

        private final HashSet<Key> keys = new HashSet<Key>();

        private final HashSet<SendableRequestItemKey> inserts = new HashSet<SendableRequestItemKey>();

        @Override

        public long checkRecentlyFailed(Key key, boolean realTime) {

            return 0;

        }

        public void addInsert(SendableRequestItemKey chosen) {

            inserts.add(chosen);

        }

        @Override

        public boolean hasKey(Key key, BaseSendableGet getterWaiting) {

            return keys.contains(key);

        }

        @Override

        public boolean hasInsert(SendableRequestItemKey token) {

            return inserts.contains(token);

        }

        public void add(Key k) {

            keys.add(k);

        }

        public void clear() {

            keys.clear();

            inserts.clear();

        }

       

    }

   

    private void testRoundTripSimpleRandom(long size, CompatibilityMode cmode) throws IOException, InsertException, MissingKeyException, FetchException, MetadataParseException, Exception {

        RandomSource r = new DummyRandomSource(12123);

        LockableRandomAccessBuffer data = generateData(r, size, smallRAFFactory);

        Bucket dataBucket = new RAFBucket(data);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        InsertContext context = baseContext.clone();

        context.earlyEncode = true;

        context.setCompatibilityMode(cmode);

        cmode = context.getCompatibilityMode();

        KeysFetchingLocally keys = new MyKeysFetchingLocally();

        boolean old = cmode.code < CompatibilityMode.COMPAT_1255.code;

        byte cryptoAlgorithm = this.cryptoAlgorithm;

        if(!(cmode == CompatibilityMode.COMPAT_CURRENT || cmode.ordinal() >= CompatibilityMode.COMPAT_1416.ordinal()))

            cryptoAlgorithm = Key.ALGO_AES_PCFB_256_SHA256;

        else

            cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256;

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, false, context,

                cryptoAlgorithm, old ? null : cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        assertTrue(storage.getStatus() == Status.ENCODED);

        // Encoded. Now try to decode it ...

        cb.waitForHasKeys();

        Metadata metadata = storage.encodeMetadata();

        // Ugly hack because Metadata behaves oddly.

        // FIXME make Metadata behave consistently and get rid.

        Bucket metaBucket = metadata.toBucket(smallBucketFactory);

        Metadata m1 = Metadata.construct(metaBucket);

        Bucket copyBucket = m1.toBucket(smallBucketFactory);

        assertTrue(BucketTools.equalBuckets(metaBucket, copyBucket));

       

        MyFetchCallback fcb = new MyFetchCallback();

       

        FetchContext fctx = HighLevelSimpleClientImpl.makeDefaultFetchContext(size*2, size*2, smallBucketFactory, new SimpleEventProducer());

       

        SplitFileFetcherStorage fetcherStorage = new SplitFileFetcherStorage(m1, fcb, new ArrayList<COMPRESSOR_TYPE>(),

                new ClientMetadata(), false, cmode.code, fctx, false, salt, URI, URI, true, new byte[0],

                r, smallBucketFactory, smallRAFFactory, jobRunner, ticker, memoryLimitedJobRunner,

                checker, false, null, null, keys);

       

        fetcherStorage.start(false);

       

        // Fully decode one segment at a time, ignore cross-segment.

       

        for(int i=0;i<storage.segments.length;i++) {

            SplitFileFetcherSegmentStorage fetcherSegment = fetcherStorage.segments[i];

            SplitFileInserterSegmentStorage inserterSegment = storage.segments[i];

            int minBlocks = inserterSegment.dataBlockCount + inserterSegment.crossCheckBlockCount;

            int totalBlocks = inserterSegment.totalBlockCount;

            boolean[] fetched = new boolean[totalBlocks];

            if(i == storage.segments.length-1 && cmode.ordinal() < CompatibilityMode.COMPAT_1255.ordinal())

                fetched[inserterSegment.dataBlockCount-1] = true; // We don't use the last block of the last segment for old splitfiles

            for(int j=0;j<minBlocks;j++) {

                int blockNo;

                do {

                    blockNo = r.nextInt(totalBlocks);

                } while (fetched[blockNo]);

                fetched[blockNo] = true;

                ClientCHKBlock block = inserterSegment.encodeBlock(blockNo);

                assertFalse(fetcherSegment.hasStartedDecode());

                boolean success = fetcherSegment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock());

                assertTrue(success);

                fcb.checkFailed();

            }

            assertTrue(fetcherSegment.hasStartedDecode());

            fcb.checkFailed();

            waitForDecode(fetcherSegment);

        }

        fcb.waitForFinished();

        verifyOutput(fetcherStorage, dataBucket);

        fetcherStorage.finishedFetcher();

        fcb.waitForFree();

    }

   

    private void testResumeCrossSegment(long size) throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        Random r = new Random(12121);

        LockableRandomAccessBuffer data = generateData(r, size, bigRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(),

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        storage.start();

        cb.waitForFinishedEncode();

        cb.waitForHasKeys();

        executor.waitForIdle();

        Metadata metadata = storage.encodeMetadata();

        assertTrue(storage.getStatus() == Status.ENCODED);

        Bucket mBucket1 = bigBucketFactory.makeBucket(-1);

        DataOutputStream os = new DataOutputStream(mBucket1.getOutputStream());

        metadata.writeTo(os);

        os.close();

        SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r,

                memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null);

        // Doesn't need to start since already encoded.

        Metadata metadata2 = storage.encodeMetadata();

        Bucket mBucket2 = bigBucketFactory.makeBucket(-1);

        os = new DataOutputStream(mBucket2.getOutputStream());

        metadata2.writeTo(os);

        os.close();

        assertTrue(BucketTools.equalBuckets(mBucket1, mBucket2));

        // Choose and succeed all blocks.

        boolean[][] chosenBlocks = new boolean[storage.segments.length][];

        for(int i=0;i<storage.segments.length;i++) {

            int blocks = storage.segments[i].totalBlockCount;

            chosenBlocks[i] = new boolean[blocks];

            assertEquals(storage.segments[i].totalBlockCount, resumed.segments[i].totalBlockCount);

        }

        int totalBlocks = storage.getTotalBlockCount();

        assertEquals(totalBlocks, resumed.getTotalBlockCount());

        for(int i=0;i<totalBlocks;i++) {

            BlockInsert chosen = resumed.chooseBlock();

            if(chosen == null) {

                assertFalse(true);

            } else {

                keys.addInsert(chosen);

            }

            assertTrue(chosen != null);

            assertFalse(chosenBlocks[chosen.segment.segNo][chosen.blockNumber]);

            chosenBlocks[chosen.segment.segNo][chosen.blockNumber] = true;

            chosen.segment.onInsertedBlock(chosen.blockNumber,

                    chosen.segment.encodeBlock(chosen.blockNumber).getClientKey());

        }

        cb.waitForSucceededInsert();

        assertEquals(Status.SUCCEEDED, resumed.getStatus());

    }

   

    private void testEncodeAfterShutdownCrossSegment(long size) throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        Random r = new Random(12121);

        LockableRandomAccessBuffer data = generateData(r, size, bigRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(),

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        executor.waitForIdle();

        // Has not encoded anything.

        for(SplitFileInserterSegmentStorage segment : storage.segments)

            assert(!segment.isFinishedEncoding());

        SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r,

                memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null);

        resumed.start();

        cb.waitForFinishedEncode();

        cb.waitForHasKeys();

        executor.waitForIdle();

        resumed.encodeMetadata();

        assertTrue(resumed.getStatus() == Status.ENCODED);

        resumed.originalData.free();

        resumed.getRAF().free();

    }

   

    private void testRepeatedEncodeAfterShutdownCrossSegment(long size) throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException {

        Random r = new Random(12121);

        LockableRandomAccessBuffer data = generateData(r, size, bigRAFFactory);

        HashResult[] hashes = getHashes(data);

        MyCallback cb = new MyCallback();

        MyKeysFetchingLocally keys = new MyKeysFetchingLocally();

        // Only enough for one segment at a time.

        MemoryLimitedJobRunner memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 1, executor);

        SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null,

                new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(),

                cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,

                r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0);

        executor.waitForIdle();

        // Has not encoded anything.

        for(SplitFileInserterSegmentStorage segment : storage.segments)

            assert(!segment.isFinishedEncoding());

        SplitFileInserterStorage resumed = null;

        if(storage.crossSegments != null) {

            for(int i=0;i<storage.crossSegments.length;i++) {

                memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 1, executor);

                resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r,

                        memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null);

                assertEquals(i, countEncodedCrossSegments(resumed));

                resumed.start();

                // The memoryLimitedJobRunner will only encode one segment at a time.

                // Wait for it to encode one segment.

                memoryLimitedJobRunner.shutdown();

                memoryLimitedJobRunner.waitForShutdown();

                executor.waitForIdle();

                assertEquals(i+1, countEncodedCrossSegments(resumed));

            }

        }

       

        for(int i=0;i<storage.segments.length;i++) {

            memoryLimitedJobRunner = new MemoryLimitedJobRunner(9*1024*1024L, 1, executor);

            resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r,

                    memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null);

            assertEquals(i, countEncodedSegments(resumed));

            if(storage.crossSegments != null) {

                assertEquals(resumed.crossSegments.length, countEncodedCrossSegments(resumed));

                assertTrue(resumed.getStatus() == Status.ENCODED_CROSS_SEGMENTS);

            }

            resumed.start();

            // The memoryLimitedJobRunner will only encode one segment at a time.

            // Wait for it to encode one segment.

            memoryLimitedJobRunner.shutdown();

            memoryLimitedJobRunner.waitForShutdown();

            executor.waitForIdle();

            assertEquals(i+1, countEncodedSegments(resumed));