Package com.asakusafw.bulkloader.bean

Examples of com.asakusafw.bulkloader.bean.ImportTargetTableBean

37
38
39
40
41
42
43
44
45
46
47
48
     */
    public void deleteFile(ImportBean bean) {
        // Import対象ファイルを削除
        List<String> list = bean.getImportTargetTableList();
        for (String tableName : list) {
            ImportTargetTableBean targetTable = bean.getTargetTable(tableName);
            File file = targetTable.getImportFile();
            // ファイルが存在する場合は削除する。
            if (file != null && file.exists()) {
                if (!file.delete()) {
                    LOG.warn("TG-IMPORTER-05001", file.getPath());
                }
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
        assert content != null;
        assert bean != null;
        assert user != null;
        String tableName = FileNameUtil.getImportTableName(protocol.getLocation());

        ImportTargetTableBean targetTableBean = bean.getTargetTable(tableName);
        if (targetTableBean == null) {
            // 対応するテーブルの定義がDSL存在しない場合異常終了する。
            throw new BulkLoaderSystemException(getClass(), "TG-EXTRACTOR-02001",
                    MessageFormat.format(
                            "エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}",
                            tableName));
        }

        URI dfsFilePath = resolveLocation(bean, user, targetTableBean.getDfsFilePath());
        Class<?> targetTableModel = targetTableBean.getImportTargetType();

        LOG.info("TG-EXTRACTOR-02002",
                tableName, dfsFilePath.toString(), targetTableModel.toString());

        // ファイルをジョブ入力データ領域に書き出す
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
            || protocol.getKind() == FileProtocol.Kind.UPDATE_CACHE;

        CacheInfo info = protocol.getInfo();
        assert info != null;

        ImportTargetTableBean targetTableBean = bean.getTargetTable(info.getTableName());
        if (targetTableBean == null) {
            // 対応するテーブルの定義がDSL存在しない場合異常終了する。
            throw new BulkLoaderSystemException(getClass(), "TG-EXTRACTOR-02001",
                    MessageFormat.format(
                            "エントリに対応するテーブルの定義がDSL存在しない。テーブル名：{0}",
                            info.getTableName()));
        }

        URI dfsFilePath = resolveLocation(bean, user, protocol.getLocation());
        try {
            CacheStorage storage = new CacheStorage(new Configuration(), dfsFilePath);
            try {
                LOG.info("TG-EXTRACTOR-11001", info.getId(), info.getTableName(), storage.getPatchProperties());
                storage.putPatchCacheInfo(info);
                LOG.info("TG-EXTRACTOR-11002", info.getId(), info.getTableName(), storage.getPatchProperties());

                Class<?> targetTableModel = targetTableBean.getImportTargetType();
                Path targetUri = storage.getPatchContents("0");
                LOG.info("TG-EXTRACTOR-11003", info.getId(), info.getTableName(), targetUri);
                long recordCount = write(targetTableModel, targetUri.toUri(), content);
                LOG.info("TG-EXTRACTOR-11004", info.getId(), info.getTableName(), targetUri, recordCount);
                LOG.info("TG-PROFILE-01002",
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
        JobFlowParamLoader loader = new JobFlowParamLoader();
        if (loader.loadCacheBuildParam(targetName, batchId, jobflowId) == false) {
            return null;
        }
        ImportTargetTableBean table = null;
        for (Map.Entry<String, ImportTargetTableBean> entry : loader.getImportTargetTables().entrySet()) {
            if (entry.getKey().equals(tableName)) {
                table = entry.getValue();
                break;
            }
        }
        if (table == null) {
            LOG.error("TG-BUILDCACHE-01005", targetName, batchId, jobflowId, executionId, tableName);
            return null;
        }
        if (table.getCacheId() == null) {
            LOG.error("TG-BUILDCACHE-01006", targetName, batchId, jobflowId, executionId, tableName);
            return null;
        }
        bean.setTargetTable(Collections.singletonMap(tableName, table));
        return bean;
View Full Code Here
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
        // Import設定の検証
        // 実行結果を検証
        assertTrue(result);
        // テーブルXXXの検証
        ImportTargetTableBean impTable1 = importTargetTable.get("XXX");
        // Import対象カラム
        assertEquals(3, impTable1.getImportTargetColumns().size());
        assertEquals("columnA", impTable1.getImportTargetColumns().get(0));
        assertEquals("columnB", impTable1.getImportTargetColumns().get(1));
        assertEquals("columnC", impTable1.getImportTargetColumns().get(2));
        // 検索条件
        assertEquals("columnA='1' and columnB='2'", impTable1.getSearchCondition());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("1"), impTable1.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("3"), impTable1.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ImportTargetTableBean", impTable1.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/${user}/${execution_id}/import_target1", impTable1.getDfsFilePath());

        // テーブルYYYの検証
        ImportTargetTableBean impTable2 = importTargetTable.get("YYY");
        // Import対象カラム
        assertEquals(1, impTable2.getImportTargetColumns().size());
        assertEquals("columnA", impTable2.getImportTargetColumns().get(0));
        // 検索条件
        assertEquals("columnA='1' or columnB=(select columnB from tableA where column='3')", impTable2.getSearchCondition());
        // キャッシュ利用有無
        assertEquals(false, impTable2.isUseCache());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("2"), impTable2.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("1"), impTable2.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ImportBean", impTable2.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/asakusa/import/11/YYY", impTable2.getDfsFilePath());

        // テーブルZZZの検証
        ImportTargetTableBean impTable3 = importTargetTable.get("ZZZ");
        // Import対象カラム
        assertEquals(2, impTable3.getImportTargetColumns().size());
        assertEquals("columnA", impTable3.getImportTargetColumns().get(0));
        assertEquals("columnB", impTable3.getImportTargetColumns().get(1));
        // 検索条件
        assertNull(impTable3.getSearchCondition());
        // キャッシュ利用有無
        assertEquals(false, impTable3.isUseCache());
        // ロック取得タイプ
        assertEquals(ImportTableLockType.find("3"), impTable3.getLockType());
        // ロック済みの場合の取り扱い
        assertEquals(ImportTableLockedOperation.find("2"), impTable3.getLockedOperation());
        // JavaBeansクラス名
        assertEquals("com.asakusafw.bulkloader.bean.ExporterBean", impTable3.getImportTargetType().getName());
        // HDFS上の出力パス
        assertEquals("/asakusa/import/11/ZZZ", impTable3.getDfsFilePath());

        // Export設定の検証
        // テーブルXXXの検証
        ExportTargetTableBean expTable1 = exportTargetTable.get("XXX");
        // 重複チェックを行うか否か
View Full Code Here
2629
2630
2631
2632
2633
2634
2635
2636
2637
            }
        };
        boolean result = loader.loadCacheBuildParam(targetName, "batch01", "11");
        assertThat(result, is(true));
        assertThat(loader.getImportTargetTables().size(), is(2));
        ImportTargetTableBean target = loader.getImportTargetTables().get("XXX");
        assertThat(target.getCacheId(), is(notNullValue()));
        assertThat(target.getLockType(), is(ImportTableLockType.NONE));
    }
View Full Code Here
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
    public void checkImportParamTest02() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.NONE);
        tableBean1.setLockedOperation(ImportTableLockedOperation.FORCE);
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        ImportTargetTableBean tableBean2 = new ImportTargetTableBean();
        tableBean2.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA2"}));
        tableBean2.setSearchCondition("TEXTDATA2='testdata2-3'");
        tableBean2.setUseCache(false);
        tableBean2.setLockType(ImportTableLockType.find("1"));
        tableBean2.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean2.setImportTargetType(this.getClass());
        tableBean2.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET2", tableBean2);


        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", null}));

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1"}));
        tableBean2.setImportTargetColumns(Arrays.asList(new String[]{""}));

        // テスト実行3
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
    public void checkImportParamTest03() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("0"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("1"));
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setLockType(null);

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
    public void checkImportParamTest04() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("1"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("0"));
        tableBean1.setImportTargetType(this.getClass());
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行1
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);

        // 設定変更
        tableBean1.setLockedOperation(null);

        // テスト実行2
        result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
    }
View Full Code Here
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
    public void checkImportParamTest05() throws Exception {
        JobFlowParamLoader loader = new JobFlowParamLoader();

        // 設定
        Map<String, ImportTargetTableBean> targetTable = new LinkedHashMap<String, ImportTargetTableBean>();
        ImportTargetTableBean tableBean1 = new ImportTargetTableBean();
        tableBean1.setImportTargetColumns(Arrays.asList(new String[]{"TEXTDATA1", "INTDATA1", "DATEDATA1"}));
        tableBean1.setSearchCondition("INTDATA1=11");
        tableBean1.setUseCache(false);
        tableBean1.setLockType(ImportTableLockType.find("1"));
        tableBean1.setLockedOperation(ImportTableLockedOperation.find("3"));
        tableBean1.setImportTargetType(null);
        tableBean1.setDfsFilePath("hdfs://localhost/user/asakusa/import/11/XXX_1");
        targetTable.put("IMPORT_TARGET1", tableBean1);

        // テスト実行
        boolean result = loader.checkImportParam(targetTable, targetName, "1", "dummyFileName", true);
        assertFalse(result);
View Full Code Here
TOP

Related Classes of com.asakusafw.bulkloader.bean.ImportTargetTableBean

  • com.asakusafw.bulkloader.cache.BuildCache
  • com.asakusafw.bulkloader.cache.GetCacheInfoLocal
  • com.asakusafw.bulkloader.cache.GetCacheInfoLocalTest
  • com.asakusafw.bulkloader.common.JobFlowParamLoader
  • com.asakusafw.bulkloader.common.JobFlowParamLoaderTest
  • com.asakusafw.bulkloader.exporter.LockReleaseTest
  • com.asakusafw.bulkloader.extractor.DfsFileImport
  • com.asakusafw.bulkloader.extractor.DfsFileImportTest
  • com.asakusafw.bulkloader.importer.ImportFileCreate
  • com.asakusafw.bulkloader.importer.ImportFileCreateTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.