Package org.apache.hadoop.tools

Examples of org.apache.hadoop.tools.DistCpOptions

109
110
111
112
113
114
115
116
117
118
119
120
121
    protected DistCpOptions getDistCpOptions(CommandLine cmd) {
        String[] paths = cmd.getOptionValue("sourcePaths").trim().split(",");
        List<Path> srcPaths = getPaths(paths);
        String trgPath = cmd.getOptionValue("targetPath").trim();

        DistCpOptions distcpOptions = new DistCpOptions(srcPaths, new Path(trgPath));
        distcpOptions.setSyncFolder(true);
        distcpOptions.setBlocking(true);
        distcpOptions.setMaxMaps(Integer.valueOf(cmd.getOptionValue("maxMaps")));
        distcpOptions.setMapBandwidthKB(Integer.valueOf(cmd.getOptionValue("mapBandwidthKB")));

        return distcpOptions;
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
        };

        FeedReplicator replicator = new FeedReplicator();
        CommandLine cmd = replicator.getCommand(args);
        DistCpOptions options = replicator.getDistCpOptions(cmd);

        List<Path> srcPaths = new ArrayList<Path>();
        srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
        Assert.assertEquals(options.getMaxMaps(), 3);
        Assert.assertEquals(options.getMapBandwidth(), 4096);
        Assert.assertEquals(options.getSourcePaths(), srcPaths);
        Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
      FsPermission sourcePerm = new FsPermission((short) 511);
      FsPermission initialPerm = new FsPermission((short) 448);
      sourceBase = TestDistCpUtils.createTestSetup(fs, sourcePerm);
      targetBase = TestDistCpUtils.createTestSetup(fs, initialPerm);

      DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
          new Path("/out"));
      options.preserve(FileAttribute.PERMISSION);
      options.appendToConf(conf);
      options.setTargetPathExists(false);
     
      CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
      Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
      listing.buildListing(listingFile, options);
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
      sourceBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
      targetBase = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
      String targetBaseAdd = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault());
      fs.rename(new Path(targetBaseAdd), new Path(targetBase));

      DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
          new Path("/out"));
      options.setSyncFolder(true);
      options.setDeleteMissing(true);
      options.appendToConf(conf);

      CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
      Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
      listing.buildListing(listingFile, options);
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
      TestDistCpUtils.createFile(fs, targetBase + "/5");
      TestDistCpUtils.createFile(fs, targetBase + "/7");
      TestDistCpUtils.createFile(fs, targetBase + "/9");
      TestDistCpUtils.createFile(fs, targetBase + "/A");

      DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)),
          new Path("/out"));
      options.setSyncFolder(true);
      options.setDeleteMissing(true);
      options.appendToConf(conf);

      CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
      Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
      listing.buildListing(listingFile, options);
View Full Code Here
72
73
74
75
76
77
78
79
80
    Path targetPath = new Path(cluster.getFileSystem().getUri().toString()
                               + "/tmp/target");

    List<Path> sourceList = new ArrayList<Path>();
    sourceList.add(sourcePath);
    final DistCpOptions distCpOptions = new DistCpOptions(sourceList, targetPath);
    distCpOptions.setMaxMaps(nMaps);
    return distCpOptions;
  }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  public static void tearDown() {
    cluster.shutdown();
  }

  public void testGetSplits(int nMaps) throws Exception {
    DistCpOptions options = getOptions(nMaps);
    Configuration configuration = new Configuration();
    configuration.set("mapred.map.tasks",
                      String.valueOf(options.getMaxMaps()));
    Path listFile = new Path(cluster.getFileSystem().getUri().toString()
        + "/tmp/testGetSplits_1/fileList.seq");
    CopyListing.getCopyListing(configuration, CREDENTIALS, options).
        buildListing(listFile, options);
View Full Code Here
81
82
83
84
85
86
87
88
89
    Path targetPath = new Path(cluster.getFileSystem().getUri().toString()
            + "/tmp/target");

    List<Path> sourceList = new ArrayList<Path>();
    sourceList.add(sourcePath);
    DistCpOptions options = new DistCpOptions(sourceList, targetPath);
    options.setMaxMaps(NUM_SPLITS);
    return options;
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    cluster.shutdown();
  }

  @Test
  public void testGetSplits() throws Exception {
    DistCpOptions options = getOptions();
    Configuration configuration = new Configuration();
    configuration.set("mapred.map.tasks",
                      String.valueOf(options.getMaxMaps()));
    CopyListing.getCopyListing(configuration, CREDENTIALS, options).buildListing(
            new Path(cluster.getFileSystem().getUri().toString()
                    +"/tmp/testDynInputFormat/fileList.seq"), options);

    JobContext jobContext = new JobContextImpl(configuration, new JobID());
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
public class TestOptionsParser {

  @Test
  public void testParseIgnoreFailure() {
    DistCpOptions options = OptionsParser.parse(new String[] {
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertFalse(options.shouldIgnoreFailures());

    options = OptionsParser.parse(new String[] {
        "-i",
        "hdfs://localhost:8020/source/first",
        "hdfs://localhost:8020/target/"});
    Assert.assertTrue(options.shouldIgnoreFailures());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.tools.DistCpOptions

  • org.apache.falcon.replication.FeedReplicator
  • org.apache.falcon.replication.FeedReplicatorTest
  • org.apache.falcon.replication.FilteredCopyListingTest
  • org.apache.hadoop.tools.mapred.lib.TestDynamicInputFormat
  • org.apache.hadoop.tools.mapred.TestCopyCommitter
  • org.apache.hadoop.tools.mapred.TestUniformSizeInputFormat
  • org.apache.hadoop.tools.TestOptionsParser
  • org.apache.ivory.repliation.FeedReplicatorTest
  • org.apache.ivory.replication.FeedReplicator
  • org.apache.ivory.replication.FilteredCopyListingTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.