Package com.asakusafw.thundergate.runtime.cache

Examples of com.asakusafw.thundergate.runtime.cache.CacheStorage


    private CacheInfo getCacheInfo(String location) throws BulkLoaderSystemException {
        assert location != null;
        URI cacheBaseUri = FileNameUtil.createPath(getConf(), location, executionId, userName).toUri();
        try {
            CacheStorage storage = new CacheStorage(getConf(), cacheBaseUri);
            try {
                if (RuntimeContext.get().canExecute(storage)) {
                    return storage.getHeadCacheInfo();
                } else {
                    return null;
                }
            } finally {
                IOUtils.closeQuietly(storage);
View Full Code Here


                            info.getTableName()));
        }

        URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
        try {
            CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath);
            try {
                LOG.info("TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
                storage.putPatchCacheInfo(info);
                LOG.info("TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());

                Class<?> targetTableModel = targetTableBean.getImportTargetType();
                Path targetUri = storage.getPatchContents("0");
                LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
                long recordCount = write(targetTableModel, targetUri.toUri(), content);
                LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
                LOG.info("TG-PROFILE-01002",
                        bean.getTargetName(),
                        bean.getBatchId(),
                        bean.getJobflowId(),
                        bean.getExecutionId(),
                        info.getTableName(),
                        recordCount);
                return recordCount;
            } finally {
                storage.close();
            }
        } catch (IOException e) {
            throw new BulkLoaderSystemException(e, getClass(), "TG-EXTRACTOR-11005",
                    info.getId(), info.getTableName(), dfsFilePath);
        }
View Full Code Here

                    Arrays.toString(args)));
        }

        Path cacheDirectory = new Path(args[1]);
        modelClass = getConf().getClassByName(args[2]);
        this.storage = new CacheStorage(getConf(), cacheDirectory.toUri());
        try {
            clearNext();
            if (create) {
                create();
            } else {
View Full Code Here

                "EXAMPLE",
                Collections.singleton("COL"),
                "com.example.Model",
                123L);
        framework.deployLibrary(TestDataModel.class, "batchapps/tbatch/lib/jobflow-tflow.jar");
        CacheStorage storage = new CacheStorage(getConfiguration(), getTargetUri());
        try {
            storage.putPatchCacheInfo(info);
            ModelOutput<TestDataModel> head = create(storage, storage.getHeadContents("0"));
            try {
                TestDataModel model = new TestDataModel();
                for (int i = 0; i < 10; i++) {
                    model.systemId.set(i);
                    model.value.set("HEAD");
                    model.deleted.set(false);
                    head.write(model);
                }
            } finally {
                head.close();
            }
            ModelOutput<TestDataModel> patch = create(storage, storage.getPatchContents("0"));
            try {
                TestDataModel model = new TestDataModel();
                for (int i = 0; i < 10; i += 2) {
                    model.systemId.set(i);
                    model.value.set("NEXT");
                    model.deleted.set(i % 4 == 0);
                    patch.write(model);
                }
            } finally {
                patch.close();
            }

            execute(CacheBuildClient.SUBCOMMAND_UPDATE);
            assertThat(storage.getHeadCacheInfo(), is(info));

            List<TestDataModel> results = collect(storage, storage.getHeadContents("*"));
            assertThat(results.size(), is(7));
            assertThat(results.get(0).systemId.get(), is(1L));
            assertThat(results.get(0).value.toString(), is("HEAD"));
            assertThat(results.get(1).systemId.get(), is(2L));
            assertThat(results.get(1).value.toString(), is("NEXT"));
            assertThat(results.get(2).systemId.get(), is(3L));
            assertThat(results.get(2).value.toString(), is("HEAD"));
            assertThat(results.get(3).systemId.get(), is(5L));
            assertThat(results.get(3).value.toString(), is("HEAD"));
            assertThat(results.get(4).systemId.get(), is(6L));
            assertThat(results.get(4).value.toString(), is("NEXT"));
            assertThat(results.get(5).systemId.get(), is(7L));
            assertThat(results.get(5).value.toString(), is("HEAD"));
            assertThat(results.get(6).systemId.get(), is(9L));
            assertThat(results.get(6).value.toString(), is("HEAD"));
        } finally {
            storage.close();
        }
    }
View Full Code Here

        assertThat(results.get(info3.getPath()), is(Kind.RESPONSE_DELETED));
    }

    private void prepare(LocalCacheInfo... caches) throws IOException {
        for (LocalCacheInfo info : caches) {
            CacheStorage storage = new CacheStorage(remote.getConf(), uri(info.getId()));
            try {
                storage.getFileSystem().create(storage.getHeadContents("0")).close();
            } finally {
                storage.close();
            }
        }
    }
View Full Code Here

                "available",
                Collections.singleton("COL"),
                "com.example.Model",
                123L);

        CacheStorage storage = new CacheStorage(remote.getConf(), uri("available"));
        try {
            storage.putHeadCacheInfo(info);
        } finally {
            storage.close();
        }

        ImportBean bean = createBean();
        Map<String, ImportTargetTableBean> map = new HashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean table = new ImportTargetTableBean();
View Full Code Here

                "available",
                Collections.singleton("COL"),
                "com.example.Model",
                123L);

        CacheStorage storage = new CacheStorage(remote.getConf(), uri("available"));
        try {
            storage.putHeadCacheInfo(info);
        } finally {
            storage.close();
        }

        ImportBean bean = createBean();
        Map<String, ImportTargetTableBean> map = new HashMap<String, ImportTargetTableBean>();
View Full Code Here

                "EXAMPLE",
                Collections.singleton("COL"),
                "com.example.Model",
                123L);

        CacheStorage storage = new CacheStorage(service.getConf(), uri("available"));
        try {
            storage.putHeadCacheInfo(info);
        } finally {
            storage.close();
        }

        FileList.Reader reader = prepare("available");
        FileList.Writer writer = FileList.createWriter(writerBuffer, false);
        service.execute(reader, writer);
View Full Code Here

                "EXAMPLE",
                Collections.singleton("COL"),
                "com.example.Model",
                123L);

        CacheStorage storage = new CacheStorage(service.getConf(), uri("available"));
        try {
            storage.putHeadCacheInfo(info);
        } finally {
            storage.close();
        }

        FileList.Reader reader = prepare("nothing1", "nothing2", "available", "nothing3");
        FileList.Writer writer = FileList.createWriter(writerBuffer, false);
        service.execute(reader, writer);
View Full Code Here

     * cache information exists but was broken.
     * @throws Exception if failed
     */
    @Test
    public void broken() throws Exception {
        CacheStorage storage = new CacheStorage(service.getConf(), uri("available"));
        try {
            // empty file
            FSDataOutputStream file = storage.getFileSystem().create(storage.getHeadProperties());
            file.close();
        } finally {
            storage.close();
        }

        FileList.Reader reader = prepare("available");
        FileList.Writer writer = FileList.createWriter(writerBuffer, false);
        service.execute(reader, writer);
View Full Code Here

TOP

Related Classes of com.asakusafw.thundergate.runtime.cache.CacheStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.