Package com.alibaba.otter.node.etl.common.pipe.impl.http.archive

Examples of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean$ArchiveEntry


public class ArchiveBeanIntegration extends BaseOtterTest {

    // @Test
    public void test_pack() {
        ArchiveBean archiveBean = new ArchiveBean();
        try {
            archiveBean.afterPropertiesSet();
            archiveBean.setUseLocalFileMutliThread(false);
        } catch (Exception e1) {
            want.fail();
        }

        File file = new File("/tmp/otter/test");
        Collection<File> allFiles = FileUtils.listFiles(file, new String[] { "jpg" }, true);

        List<FileData> fileDatas = new ArrayList<FileData>();
        for (File files : allFiles) {
            FileData data = new FileData();
            // data.setPath("wsproduct_repository/product_sku/76/84/32/84/768432847_10.summ.jpg");
            data.setPath(StringUtils.substringAfter(files.getAbsolutePath(), "/tmp/otter/test"));
            fileDatas.add(data);
        }

        File archiveFile = new File("/tmp/otter/test.gzip");
        if (archiveFile.exists()) {
            archiveFile.delete();
        }

        boolean result = archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

            public InputStream retrive(FileData source) {
                return new LazyFileInputStream(new File("/tmp/otter/test", source.getPath()));
            }
        });
View Full Code Here


    }

    @Test
    public void test_unpack() {
        ArchiveBean archiveBean = new ArchiveBean();
        try {
            archiveBean.afterPropertiesSet();
            archiveBean.setUseLocalFileMutliThread(false);
        } catch (Exception e1) {
            want.fail();
        }

        File archiveFile = new File("/tmp/otter/test.gzip");
        // for (File archiveFile : archiveFiles.listFiles()) {
        // if (archiveFile.isDirectory()) {
        // continue;
        // }
        //
        // if (!"FileBatch-2013-03-07-16-27-05-245-369-3209577.gzip".equals(archiveFile.getName())) {
        // continue;
        // }

        System.out.println("start unpack : " + archiveFile.getName());
        File targetFile = new File("/tmp/otter", FilenameUtils.removeExtension(archiveFile.getName()));
        archiveBean.unpack(archiveFile, targetFile);
        // }
    }
View Full Code Here

        List<FileData> fileDatas = new ArrayList<FileData>();
        File archiveFile = new File(tmp, "pack.zip");
        File unpack = new File(tmp, "unpack");

        ArchiveBean archiveBean = new ArchiveBean();
        try {
            archiveBean.afterPropertiesSet();
        } catch (Exception e1) {
            want.fail();
        }

        try {
            // for (int i = 0; i < 10; i++) {
            // files[i] = new File(tmp, "archiveTest_" + i + ".txt");
            // datas[i] = getBlock((i + 1) * 1024);
            // NioUtils.write(datas[i], files[i]);
            //
            // FileData filedata = new FileData();
            // filedata.setPath(files[i].getPath());
            // fileDatas.add(filedata);
            // }

            archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

                public InputStream retrive(FileData source) {
                    try {
                        return new FileInputStream(new File(source.getPath()));
                    } catch (FileNotFoundException e) {
                        e.printStackTrace();
                        want.fail();
                    }
                    return null;
                }
            });

            // 开始解压
            List<File> result = archiveBean.unpack(archiveFile, unpack);
            want.bool(result.size() == fileDatas.size());

            // File dir = new File(unpack, archiveFile.getParent());
            // File[] unpackFiles = dir.listFiles();
            //
View Full Code Here

        // 压缩对应的文件数据
        List<FileData> fileDatas = fileBatch.getFiles();
        Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
        int poolSize = pipeline.getParameters().getFileLoadPoolSize();
        boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread();
        ArchiveBean archiveBean = getArchiveBean();
        archiveBean.adjustPoolSize(poolSize);// 调整线程池大小
        archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);// 设置是否启用local多线程同步
        boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() {

            public InputStream retrive(FileData fileData) {
                boolean miss = false;
                try {
                    if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
View Full Code Here

TOP

Related Classes of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean$ArchiveEntry

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.