Package com.asakusafw.bulkloader.transfer

Examples of com.asakusafw.bulkloader.transfer.FileListProvider


            // ディレクトリが存在しない場合は異常終了する。
            LOG.error("TG-EXPORTER-02001", fileDirectry.getAbsolutePath());
            return false;
        }

        FileListProvider provider = null;
        FileList.Reader reader = null;
        long totalStartTime = System.currentTimeMillis();
        try {
            provider = openFileList(
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId());
            provider.discardWriter();
            reader = provider.openReader();

            // FileListの終端まで繰り返す
            int fileSeq = 0;

            // プロファイル用のテーブル
            Map<String, TableTransferProfile> profiles = new TreeMap<String, TableTransferProfile>();

            while (reader.next()) {
                FileProtocol protocol = reader.getCurrentProtocol();
                assert protocol.getKind() == FileProtocol.Kind.CONTENT;

                // ファイル名を取得
                String fileName = protocol.getLocation();

                // テーブル名を取得
                String tableName = FileNameUtil.getExportTableName(fileName);
                if (tableName == null) {
                    LOG.error("TG-EXPORTER-02003", fileName, "(Unknown)");
                    return false;
                } else if (bean.getExportTargetTable(tableName) == null) {
                    LOG.error("TG-EXPORTER-02003", fileName, tableName);
                    return false;
                }

                // プロファイル情報の引当
                TableTransferProfile profile = profiles.get(tableName);
                if (profile == null) {
                    profile = new TableTransferProfile(tableName);
                    profiles.put(tableName, profile);
                }

                // ファイル名を作成
                File file = FileNameUtil.createExportFilePath(
                        fileDirectry,
                        bean.getTargetName(),
                        bean.getJobflowId(),
                        bean.getExecutionId(),
                        tableName,
                        fileSeq++);

                // ファイルを読み込んでローカルファイルに書き込む
                LOG.info("TG-EXPORTER-02008", tableName, file.getAbsolutePath());

                long dumpStartTime = System.currentTimeMillis();
                long dumpFileSize = 0;

                int byteSize = Integer.parseInt(
                        ConfigurationLoader.getProperty(Constants.PROP_KEY_EXP_FILE_COMP_BUFSIZE));
                byte[] b = new byte[byteSize];

                InputStream content = reader.openContent();
                OutputStream fos = null;
                try {
                    fos = createFos(file);
                    while (true) {
                        int read;
                        try {
                            read = content.read(b);
                        } catch (IOException e) {
                            throw new BulkLoaderSystemException(e, getClass(), "TG-EXPORTER-02002",
                                    "Exportファイルの読み込みに失敗。エントリ名:" + protocol.getLocation());
                        }
                        // 入力ファイルの終端を察知する
                        if (read < 0) {
                            break;
                        }
                        dumpFileSize += read;
                        // ファイルを書き出す
                        try {
                            fos.write(b, 0, read);
                        } catch (IOException e) {
                            throw new BulkLoaderSystemException(e, getClass(), "TG-EXPORTER-02002",
                                    "Exportファイルの書き出しに失敗。ファイル名:" +  file.getName());
                        }
                    }
                    // ファイル名を設定する
                    bean.getExportTargetTable(tableName).addExportFile(file);

                    // プロファイル情報を加算する
                    profile.elapsedTime += System.currentTimeMillis() - dumpStartTime;
                    profile.fileSize += dumpFileSize;

                    LOG.info("TG-EXPORTER-02009",
                            tableName, file.getAbsolutePath());

                } finally {
                    try {
                        content.close();
                    } catch (IOException e) {
                        // ここで例外が発生した場合は握りつぶす
                        e.printStackTrace();
                    }
                    if (fos != null) {
                        try {
                            fos.close();
                        } catch (IOException e) {
                            // ここで例外が発生した場合は握りつぶす
                            e.printStackTrace();
                        }
                    }
                }
            }
            for (TableTransferProfile profile : profiles.values()) {
                LOG.info("TG-PROFILE-02004",
                        bean.getTargetName(),
                        bean.getBatchId(),
                        bean.getJobflowId(),
                        bean.getExecutionId(),
                        profile.tableName,
                        profile.fileSize,
                        profile.elapsedTime);
            }
            reader.close();
            provider.waitForComplete();
            LOG.info("TG-PROFILE-02002",
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId(),
                    reader.getByteCount(),
                    System.currentTimeMillis() - totalStartTime);
        } catch (BulkLoaderSystemException e) {
            LOG.log(e);
            return false;
        } catch (Exception e) {
            LOG.error(e, "TG-EXPORTER-02002",
                    "Exportファイルの読み込みに失敗。");
            return false;
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
            if (provider != null) {
                try {
                    provider.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
        }
View Full Code Here


            return Collections.emptyMap();
        }
        LOG.info("TG-GCCACHE-02001",
                targetName,
                list.size());
        FileListProvider provider = null;
        try {
            provider = openFileList(targetName);
            Future<Void> upstream = submitUpstream(list, provider);
            Future<Map<String, FileProtocol.Kind>> downstream = submitDownstream(provider);
            Map<String, FileProtocol.Kind> results;
            while (true) {
                try {
                    if (upstream.isDone()) {
                        upstream.get();
                    }
                    results = downstream.get(1, TimeUnit.SECONDS);
                    break;
                } catch (TimeoutException e) {
                    // will retry
                } catch (CancellationException e) {
                    upstream.cancel(true);
                    downstream.cancel(true);
                    throw new IOException("Deleting cache storages was cancelled", e);
                } catch (ExecutionException e) {
                    upstream.cancel(true);
                    downstream.cancel(true);
                    Throwable cause = e.getCause();
                    if (cause instanceof Error) {
                        throw (Error) cause;
                    } else if (cause instanceof RuntimeException) {
                        throw (RuntimeException) cause;
                    } else if (cause instanceof IOException) {
                        throw (IOException) cause;
                    } else {
                        throw new AssertionError(cause);
                    }
                }
            }
            provider.waitForComplete();
            reportResults(targetName, results);
            return results;
        } catch (IOException e) {
            throw new BulkLoaderSystemException(e, getClass(), "TG-GCCACHE-02004",
                    targetName);
        } catch (InterruptedException e) {
            throw new BulkLoaderSystemException(e, getClass(), "TG-GCCACHE-02004",
                    targetName);
        } finally {
            if (provider != null) {
                try {
                    provider.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
        }
View Full Code Here

            storage.close();
        }
    }

    private void execute(String subcommand) throws IOException, InterruptedException {
        FileListProvider provider = execute(
                Constants.PATH_REMOTE_ROOT + Constants.PATH_LOCAL_CACHE_BUILD,
                subcommand,
                "tbatch",
                "tflow",
                "testing",
                getTargetUri().toString(),
                TestDataModel.class.getName());
        try {
            provider.discardReader();
            provider.discardWriter();
            provider.waitForComplete();
        } finally {
            provider.close();
        }
    }
View Full Code Here

        LOG.info("TG-IMPORTER-12001",
                bean.getTargetName(),
                bean.getBatchId(),
                bean.getJobflowId(),
                bean.getExecutionId());
        FileListProvider provider = null;
        try {
            provider = openFileList(
                    bean.getTargetName(), bean.getBatchId(), bean.getJobflowId(), bean.getExecutionId());
            Future<Void> upstream = submitUpstream(bean, provider);
            Future<Map<String, CacheInfo>> downstream = submitDownstream(provider);
            Map<String, CacheInfo> result;
            while (true) {
                try {
                    if (upstream.isDone()) {
                        upstream.get();
                    }
                    result = downstream.get(1, TimeUnit.SECONDS);
                    break;
                } catch (TimeoutException e) {
                    // will retry
                } catch (CancellationException e) {
                    upstream.cancel(true);
                    downstream.cancel(true);
                    throw new IOException("Collecting cache information was cancelled", e);
                } catch (ExecutionException e) {
                    upstream.cancel(true);
                    downstream.cancel(true);
                    Throwable cause = e.getCause();
                    if (cause instanceof Error) {
                        throw (Error) cause;
                    } else if (cause instanceof RuntimeException) {
                        throw (RuntimeException) cause;
                    } else if (cause instanceof IOException) {
                        throw (IOException) cause;
                    } else {
                        throw new AssertionError(cause);
                    }
                }
            }
            provider.waitForComplete();
            LOG.info("TG-IMPORTER-12003",
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId(),
                    result.size());
            return result;
        } catch (IOException e) {
            throw new BulkLoaderSystemException(e, getClass(), "TG-IMPORTER-12004",
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId());
        } catch (InterruptedException e) {
            throw new BulkLoaderSystemException(e, getClass(), "TG-IMPORTER-12004",
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId());
        } finally {
            if (provider != null) {
                try {
                    provider.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
        }
View Full Code Here

    public boolean sendImportFile(ImportBean bean) {
        // ZIP圧縮に関する情報を取得
        String strCompType = ConfigurationLoader.getProperty(Constants.PROP_KEY_IMP_FILE_COMP_TYPE);
        FileCompType compType = FileCompType.find(strCompType);

        FileListProvider provider = null;
        FileList.Writer writer = null;

        long totalStartTime = System.currentTimeMillis();
        try {
            provider = openFileList(
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId());
            provider.discardReader();
            writer = provider.openWriter(compType == FileCompType.DEFLATED);

            // Import対象テーブル毎にファイルの読み込み・書き出しの処理を行う
            List<String> list = arrangeSendOrder(bean);
            for (String tableName : list) {
                long tableStartTime = System.currentTimeMillis();
                ImportTargetTableBean targetTable = bean.getTargetTable(tableName);
                LOG.info("TG-IMPORTER-04004",
                        tableName,
                        targetTable.getImportFile().getAbsolutePath(),
                        compType.getSymbol());
                long dumpFileSize = sendTableFile(writer, tableName, targetTable);
                LOG.info("TG-PROFILE-02003",
                        bean.getTargetName(),
                        bean.getBatchId(),
                        bean.getJobflowId(),
                        bean.getExecutionId(),
                        tableName,
                        dumpFileSize,
                        System.currentTimeMillis() - tableStartTime);
                LOG.info("TG-IMPORTER-04005",
                        tableName,
                        targetTable.getImportFile().getAbsolutePath(),
                        compType.getSymbol());
            }
            writer.close();
            provider.waitForComplete();
            LOG.info("TG-PROFILE-02001",
                    bean.getTargetName(),
                    bean.getBatchId(),
                    bean.getJobflowId(),
                    bean.getExecutionId(),
                    writer.getByteCount(),
                    System.currentTimeMillis() - totalStartTime);
        } catch (BulkLoaderSystemException e) {
            LOG.log(e);
            return false;
        } catch (Exception e) {
            LOG.error(e, "TG-IMPORTER-04002");
            return false;
        } finally {
            if (writer != null) {
                try {
                    writer.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
            if (provider != null) {
                try {
                    provider.close();
                } catch (IOException ignored) {
                    ignored.printStackTrace();
                }
            }
        }
View Full Code Here

TOP

Related Classes of com.asakusafw.bulkloader.transfer.FileListProvider

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.