Package org.apache.hadoop.tools

Examples of org.apache.hadoop.tools.DistCpOptions

48
49
50
51
52
53
54
55
56
57
58
59
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
        };

        FeedReplicator replicator = new FeedReplicator();
        CommandLine cmd = replicator.getCommand(args);
        DistCpOptions options = replicator.getDistCpOptions(cmd);

        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
        Assert.assertEquals(options.getSourcePaths(), srcPaths);
        Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
    }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    }

    @Override
    public int run(String[] args) throws Exception {
        CommandLine cmd = getCommand(args);
        DistCpOptions options = getDistCpOptions(cmd);

        Configuration conf = this.getConf();
        // inject wf configs
        Path confPath = new Path("file:///"
                + System.getProperty("oozie.action.conf.xml"));
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
    protected DistCpOptions getDistCpOptions(CommandLine cmd) {
        String[] paths = cmd.getOptionValue("sourcePaths").trim().split(",");
        List<Path> srcPaths = getPaths(paths);
        String trgPath = cmd.getOptionValue("targetPath").trim();

        DistCpOptions distcpOptions = new DistCpOptions(srcPaths, new Path(trgPath));
        distcpOptions.setSyncFolder(true);
        distcpOptions.setBlocking(true);
        distcpOptions.setMaxMaps(Integer.valueOf(cmd.getOptionValue("maxMaps")));

        return distcpOptions;
    }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        new FilteredCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options);

        verifyContents(listingPath, -1);
    }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("falcon.include.path", "*/3/*");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("falcon.include.path", "*/3/?");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("falcon.include.path", "*/3/[47]");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("falcon.include.path", "*/3/40");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("falcon.include.path", "*/3/{4,7}");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    }

    @Override
    public int run(String[] args) throws Exception {
        CommandLine cmd = getCommand(args);
        DistCpOptions options = getDistCpOptions(cmd);

        Configuration conf = this.getConf();
        // inject wf configs
        Path confPath = new Path("file:///"
                + System.getProperty("oozie.action.conf.xml"));
View Full Code Here
TOP

Related Classes of org.apache.hadoop.tools.DistCpOptions

  • org.apache.falcon.replication.FeedReplicator
  • org.apache.falcon.replication.FeedReplicatorTest
  • org.apache.falcon.replication.FilteredCopyListingTest
  • org.apache.hadoop.tools.mapred.lib.TestDynamicInputFormat
  • org.apache.hadoop.tools.mapred.TestCopyCommitter
  • org.apache.hadoop.tools.mapred.TestUniformSizeInputFormat
  • org.apache.hadoop.tools.TestOptionsParser
  • org.apache.ivory.repliation.FeedReplicatorTest
  • org.apache.ivory.replication.FeedReplicator
  • org.apache.ivory.replication.FilteredCopyListingTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.