Package org.apache.hadoop.tools

Examples of org.apache.hadoop.tools.DistCpOptions

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
    CommandLine cmd = new GnuParser().parse(options, args);
    String[] paths = cmd.getOptionValue("sourcePaths").trim().split(",");
    List<Path> srcPaths = getPaths(paths);
    String trgPath = cmd.getOptionValue("targetPath").trim();

    DistCpOptions distcpOptions = new DistCpOptions(srcPaths, new Path(
        trgPath));
        distcpOptions.setSyncFolder(true);
    distcpOptions.setBlocking(true);
    distcpOptions.setMaxMaps(Integer.valueOf(cmd.getOptionValue("maxMaps")));
        if (cmd.hasOption("bandwidth")) {
        distcpOptions.setMapBandwidth(Integer.valueOf(cmd.getOptionValue("bandwidth")));
        } else if (cmd.hasOption("bandwidthKB")) {
            distcpOptions.setMapBandwidthKB(Integer.valueOf(cmd.getOptionValue("bandwidthKB")));
        }

    return distcpOptions;
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        new FilteredCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options);

        verifyContents(listingPath, -1);
    }
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("ivory.include.path", "*/3/*");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("ivory.include.path", "*/3/?");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("ivory.include.path", "*/3/[47]");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("ivory.include.path", "*/3/40");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
        final String pathString = uri.toString();
        Path fileSystemPath = new Path(pathString);
        Path source = new Path(fileSystemPath.toString() + "///tmp/source");
        Path target = new Path(fileSystemPath.toString() + "///tmp/target");
        Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
        DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

        Configuration configuration = new Configuration();
        configuration.set("ivory.include.path", "*/3/{4,7}");
        new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
         * <arg>-blocking</arg><arg>true</arg> <arg>-maxMaps</arg><arg>20</arg>
         * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg>
         * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg>
         */
        FeedReplicator replicator = new FeedReplicator();
        DistCpOptions options = replicator.getDistCpOptions(new String[] { "true", "-maxMaps", "5", "-sourcePaths",
                "hdfs://localhost:8020/tmp/", "-targetPath",
                "hdfs://localhost1:8020/tmp/","-bandwidth","1" });

        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
        Assert.assertEquals(options.getSourcePaths(), srcPaths);
        Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
public class TestOptionsParser {

  @Test
  public void testParseIgnoreFailure() {
    DistCpOptions options = OptionsParser.parse(new String[] {
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertFalse(options.shouldIgnoreFailures());

    options = OptionsParser.parse(new String[] {
        "-i",
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertTrue(options.shouldIgnoreFailures());
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    Assert.assertTrue(options.shouldIgnoreFailures());
  }

  @Test
  public void testParseOverwrite() {
    DistCpOptions options = OptionsParser.parse(new String[] {
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertFalse(options.shouldOverwrite());

    options = OptionsParser.parse(new String[] {
        "-overwrite",
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertTrue(options.shouldOverwrite());

    try {
      OptionsParser.parse(new String[] {
          "-update",
          "-overwrite",
View Full Code Here
TOP

Related Classes of org.apache.hadoop.tools.DistCpOptions

  • org.apache.falcon.replication.FeedReplicator
  • org.apache.falcon.replication.FeedReplicatorTest
  • org.apache.falcon.replication.FilteredCopyListingTest
  • org.apache.hadoop.tools.mapred.lib.TestDynamicInputFormat
  • org.apache.hadoop.tools.mapred.TestCopyCommitter
  • org.apache.hadoop.tools.mapred.TestUniformSizeInputFormat
  • org.apache.hadoop.tools.TestOptionsParser
  • org.apache.ivory.repliation.FeedReplicatorTest
  • org.apache.ivory.replication.FeedReplicator
  • org.apache.ivory.replication.FilteredCopyListingTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.