Package org.apache.hadoop.dfs

Examples of org.apache.hadoop.dfs.MiniDFSCluster

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  public void testMultipleCachefiles()
  {
    try {
      boolean mayExit = false;
      MiniMRCluster mr = null;
      MiniDFSCluster dfs = null;
      FileSystem fileSys = null;
      try{
        Configuration conf = new Configuration();
        dfs = new MiniDFSCluster(conf, 1, true, null);
        fileSys = dfs.getFileSystem();
        String namenode = fileSys.getName();
        mr  = new MiniMRCluster(1, namenode, 3);
        // During tests, the default Configuration will use a local mapred
        // So don't specify -config or -cluster
        String strJobtracker = "mapred.job.tracker=" + "localhost:" + mr.getJobTrackerPort();
        String strNamenode = "fs.default.name=" + namenode;
        String argv[] = new String[] {
          "-input", INPUT_FILE,
          "-output", OUTPUT_DIR,
          "-mapper", map,
          "-reducer", reduce,
          //"-verbose",
          //"-jobconf", "stream.debug=set"
          "-jobconf", strNamenode,
          "-jobconf", strJobtracker,
          "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
          "-jobconf", "mapred.child.java.opts=-Dcontrib.name=" + System.getProperty("contrib.name") + " " +
                      "-Dbuild.test=" + System.getProperty("build.test") + " " +
                      conf.get("mapred.child.java.opts",""),
          "-cacheFile", "hdfs://"+fileSys.getName()+CACHE_FILE + "#" + mapString,
          "-cacheFile", "hdfs://"+fileSys.getName()+CACHE_FILE_2 + "#" + mapString2
        };

        fileSys.delete(new Path(OUTPUT_DIR));
       
        DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
        file.writeBytes(mapString + "\n");
        file.writeBytes(mapString2 + "\n");
        file.close();
        file = fileSys.create(new Path(CACHE_FILE));
        file.writeBytes(cacheString);
        file.close();
        file = fileSys.create(new Path(CACHE_FILE_2));
        file.writeBytes(cacheString2);
        file.close();
         
        job = new StreamJob(argv, mayExit);    
        job.go();
        String line = null;
        String line2 = null;
        Path[] fileList = fileSys.listPaths(new Path(OUTPUT_DIR));
        for (int i = 0; i < fileList.length; i++){
          System.out.println(fileList[i].toString());
          BufferedReader bread =
            new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
          line = bread.readLine();
          System.out.println(line);
          line2 = bread.readLine();
          System.out.println(line2);
        }
        assertEquals(cacheString + "\t", line);
        assertEquals(cacheString2 + "\t", line2);
      } finally{
        if (fileSys != null) { fileSys.close(); }
        if (dfs != null) { dfs.shutdown(); }
        if (mr != null) { mr.shutdown();}
      }
     
    } catch(Exception e) {
      failTrace(e);
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
  public void testSymLink()
  {
    try {
      boolean mayExit = false;
      MiniMRCluster mr = null;
      MiniDFSCluster dfs = null;
      FileSystem fileSys = null;
      try{
        Configuration conf = new Configuration();
        dfs = new MiniDFSCluster(conf, 1, true, null);
        fileSys = dfs.getFileSystem();
        String namenode = fileSys.getName();
        mr  = new MiniMRCluster(1, namenode, 3);
        // During tests, the default Configuration will use a local mapred
        // So don't specify -config or -cluster
        String strJobtracker = "mapred.job.tracker=" + "localhost:" + mr.getJobTrackerPort();
        String strNamenode = "fs.default.name=" + namenode;
        String argv[] = new String[] {
          "-input", INPUT_FILE,
          "-output", OUTPUT_DIR,
          "-mapper", map,
          "-reducer", reduce,
          //"-verbose",
          //"-jobconf", "stream.debug=set"
          "-jobconf", strNamenode,
          "-jobconf", strJobtracker,
          "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
          "-jobconf", "mapred.child.java.opts=-Dcontrib.name=" + System.getProperty("contrib.name") + " " +
                      "-Dbuild.test=" + System.getProperty("build.test") + " " +
                      conf.get("mapred.child.java.opts",""),
          "-cacheFile", "hdfs://"+fileSys.getName()+CACHE_FILE + "#testlink"
        };

        fileSys.delete(new Path(OUTPUT_DIR));
       
        DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
        file.writeBytes(mapString);
        file.close();
        file = fileSys.create(new Path(CACHE_FILE));
        file.writeBytes(cacheString);
        file.close();
         
        job = new StreamJob(argv, mayExit);     
        job.go();
        String line = null;
        Path[] fileList = fileSys.listPaths(new Path(OUTPUT_DIR));
        for (int i = 0; i < fileList.length; i++){
          System.out.println(fileList[i].toString());
          BufferedReader bread =
            new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
          line = bread.readLine();
          System.out.println(line);
        }
        assertEquals(cacheString + "\t", line);
      } finally{
        if (fileSys != null) { fileSys.close(); }
        if (dfs != null) { dfs.shutdown(); }
        if (mr != null) { mr.shutdown();}
      }
     
    } catch(Exception e) {
      failTrace(e);
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
      assertTrue("Directory " + taskDirs[i] + " not found", found[i]);
    }
  }
 
  public void testWithDFS() throws IOException {
    MiniDFSCluster dfs = null;
    MiniMRCluster mr = null;
    FileSystem fileSys = null;
    try {
      final int taskTrackers = 4;

      Configuration conf = new Configuration();
      dfs = new MiniDFSCluster(conf, 4, true, null);
      fileSys = dfs.getFileSystem();
      mr = new MiniMRCluster(taskTrackers, fileSys.getName(), 1);
      double estimate = PiEstimator.launch(NUM_MAPS, NUM_SAMPLES,
                                           mr.createJobConf());
      double error = Math.abs(Math.PI - estimate);
      assertTrue("Error in PI estimation "+error+" exceeds 0.01", (error < 0.01));
      checkTaskDirectories(mr, new String[]{}, new String[]{});
         
      // Run a word count example
      JobConf jobConf = mr.createJobConf();
      // Keeping tasks that match this pattern
      jobConf.setKeepTaskFilesPattern("task_[^_]*_[0-9]*_m_000001_.*");
      TestResult result;
      final Path inDir = new Path("/testing/wc/input");
      final Path outDir = new Path("/testing/wc/output");
      result = launchWordCount(jobConf, inDir, outDir,
                               "The quick brown fox\nhas many silly\n" +
                               "red fox sox\n",
                               3, 1);
      assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
                   "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output);
      String jobid = result.job.getJobID();
      String taskid = "task_" + jobid.substring(4) + "_m_000001_0";
      checkTaskDirectories(mr, new String[]{jobid}, new String[]{taskid});
      // test with maps=0
      jobConf = mr.createJobConf();
      result = launchWordCount(jobConf, inDir, outDir, "owen is oom", 0, 1);
      assertEquals("is\t1\noom\t1\nowen\t1\n", result.output);
      // Run a job with input and output going to localfs even though the
      // default fs is hdfs.
      {
        FileSystem localfs = FileSystem.getLocal(jobConf);
        String TEST_ROOT_DIR =
          new File(System.getProperty("test.build.data","/tmp"))
          .toString().replace(' ', '+');
        Path localIn = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/in"));
        Path localOut = localfs.makeQualified
                          (new Path(TEST_ROOT_DIR + "/local/out"));
        result = launchWordCount(jobConf, localIn, localOut,
                                 "all your base belong to us", 1, 1);
        assertEquals("all\t1\nbase\t1\nbelong\t1\nto\t1\nus\t1\nyour\t1\n",
                     result.output);
        assertTrue("outputs on localfs", localfs.exists(localOut));
      }
    } finally {
      if (fileSys != null) { fileSys.close(); }
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
    }
  }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
  public void testPipes() throws IOException {
    if (System.getProperty("compile.c++") == null) {
      LOG.info("compile.c++ is not defined, so skipping TestPipes");
      return;
    }
    MiniDFSCluster dfs = null;
    MiniMRCluster mr = null;
    FileSystem fs = null;
    Path cppExamples = new Path(System.getProperty("install.c++.examples"));
    Path inputPath = new Path("/testing/in");
    Path outputPath = new Path("/testing/out");
    try {
      final int numSlaves = 2;
      Configuration conf = new Configuration();
      dfs = new MiniDFSCluster(conf, numSlaves, true, null);
      fs = dfs.getFileSystem();
      mr = new MiniMRCluster(numSlaves, fs.getName(), 1);
      writeInputFile(fs, inputPath);
      runProgram(mr, fs, new Path(cppExamples, "bin/wordcount-simple"),
                 inputPath, outputPath, 3, 2, twoSplitOutput);
      FileUtil.fullyDelete(fs, outputPath);
      assertFalse("output not cleaned up", fs.exists(outputPath));
      runProgram(mr, fs, new Path(cppExamples, "bin/wordcount-simple"),
                 inputPath, outputPath, 3, 0, noSortOutput);
      FileUtil.fullyDelete(fs, outputPath);
      assertFalse("output not cleaned up", fs.exists(outputPath));
      runProgram(mr, fs, new Path(cppExamples, "bin/wordcount-part"),
                 inputPath, outputPath, 3, 2, fixedPartitionOutput);
      runNonPipedProgram(mr, fs, new Path(cppExamples, "bin/wordcount-nopipe"));
      mr.waitUntilIdle();
    } finally {
      mr.shutdown();
      dfs.shutdown();
    }
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    return (runningJob.isSuccessful());
  }
 
  public void testJobWithDFS() throws IOException {
    String namenode = null;
    MiniDFSCluster dfs = null;
    MiniMRCluster mr = null;
    FileSystem fileSys = null;
    try {
      final int taskTrackers = 4;
      final int jobTrackerPort = 60050;
      Configuration conf = new Configuration();
      dfs = new MiniDFSCluster(conf, 1, true, null);
      fileSys = dfs.getFileSystem();
      namenode = fileSys.getName();
      mr = new MiniMRCluster(taskTrackers, namenode, 2);
      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
      JobConf jobConf = new JobConf();
      boolean result;
      result = launchJob(namenode, jobTrackerName, jobConf,
                              3, 1);
      assertTrue(result);
         
    } finally {
      if (fileSys != null) { fileSys.close(); }
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown(); }
    }
  }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
  public void testMapDebugScript(){
    try {
     
      // create configuration, dfs, file system and mapred cluster
      Configuration cnf = new Configuration();
      dfs = new MiniDFSCluster(cnf, 1, true, null);
      fileSys = dfs.getFileSystem();
      mr = new MiniMRCluster(2, fileSys.getName(), 1);
      JobConf conf = mr.createJobConf();
     
      // intialize input, output and debug directories
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
      assertTrue("outputs on localfs", localfs.exists(localOut));
    }
  }

  public void testWithDFS() throws IOException {
    MiniDFSCluster dfs = null;
    MiniMRCluster mr = null;
    FileSystem fileSys = null;
    try {
      final int taskTrackers = 4;

      Configuration conf = new Configuration();
      dfs = new MiniDFSCluster(conf, 4, true, null);
      fileSys = dfs.getFileSystem();
      mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1);

      runPI(mr, mr.createJobConf());
      runWordCount(mr, mr.createJobConf());
    } finally {
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
    }
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
  public void testSymLink()
  {
    try {
      boolean mayExit = false;
      MiniMRCluster mr = null;
      MiniDFSCluster dfs = null;
      try{
        Configuration conf = new Configuration();
        dfs = new MiniDFSCluster(conf, 1, true, null);
        FileSystem fileSys = dfs.getFileSystem();
        String namenode = fileSys.getName();
        mr  = new MiniMRCluster(1, namenode, 3);
        // During tests, the default Configuration will use a local mapred
        // So don't specify -config or -cluster
        String strJobtracker = "mapred.job.tracker=" + "localhost:" + mr.getJobTrackerPort();
        String strNamenode = "fs.default.name=" + namenode;
        String argv[] = new String[] {
          "-input", INPUT_FILE,
          "-output", OUTPUT_DIR,
          "-mapper", map,
          "-reducer", reduce,
          //"-verbose",
          //"-jobconf", "stream.debug=set"
          "-jobconf", strNamenode,
          "-jobconf", strJobtracker,
          "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
          "-jobconf", "mapred.child.java.opts=-Dcontrib.name=" + System.getProperty("contrib.name") + " " +
                      "-Dbuild.test=" + System.getProperty("build.test") + " " +
                      conf.get("mapred.child.java.opts",""),
          "-cacheFile", "hdfs://"+fileSys.getName()+CACHE_FILE + "#testlink"
        };

        fileSys.delete(new Path(OUTPUT_DIR));
       
        DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
        file.writeBytes(mapString);
        file.close();
        file = fileSys.create(new Path(CACHE_FILE));
        file.writeBytes(cacheString);
        file.close();
         
        job = new StreamJob(argv, mayExit);     
        job.go();

        fileSys = dfs.getFileSystem();
        String line = null;
        Path[] fileList = fileSys.listPaths(new Path(OUTPUT_DIR));
        for (int i = 0; i < fileList.length; i++){
          System.out.println(fileList[i].toString());
          BufferedReader bread =
            new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
          line = bread.readLine();
          System.out.println(line);
        }
        assertEquals(cacheString + "\t", line);
      } finally{
        if (dfs != null) { dfs.shutdown(); }
        if (mr != null) { mr.shutdown();}
      }
     
    } catch(Exception e) {
      failTrace(e);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    CACHE_FILE_2 = new File("cacheArchive2");
    input = "HADOOP";
    expectedOutput = "HADOOP\t\nHADOOP\t\n";
    try {
      conf = new Configuration();     
      dfs = new MiniDFSCluster(conf, 1, true, null);     
      fileSys = dfs.getFileSystem();
      namenode = fileSys.getUri().getAuthority();
      mr  = new MiniMRCluster(1, namenode, 3);
      strJobTracker = "mapred.job.tracker=" + "localhost:" + mr.getJobTrackerPort();
      strNamenode = "fs.default.name=" + namenode;
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    boolean success = false;
    String base = new File(".").getAbsolutePath();
    System.setProperty("hadoop.log.dir", base + "/logs");
    conf_ = new Configuration();
    String overrideFS = StreamUtil.getBoundAntProperty("fs.default.name", null);
    MiniDFSCluster cluster = null;
    try {
      if (overrideFS == null) {
        cluster = new MiniDFSCluster(conf_, 1, true, null);
        fs_ = cluster.getFileSystem();
      } else {
        System.out.println("overrideFS: " + overrideFS);
        conf_.set("fs.default.name", overrideFS);
        fs_ = FileSystem.get(conf_);
      }
      doAllTestJobs();
      success = true;
    } catch (IOException io) {
      io.printStackTrace();
    } finally {
      try {
        fs_.close();
      } catch (IOException io) {
      }
      if (cluster != null) {
        cluster.shutdown();
        System.out.println("cluster.shutdown(); DONE");
      }
      System.out.println(getClass().getName() + ": success=" + success);
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.dfs.MiniDFSCluster

  • org.apache.hadoop.cli.TestCLI
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.contrib.index.mapred.TestDistributionPolicy
  • org.apache.hadoop.contrib.index.mapred.TestIndexUpdater
  • org.apache.hadoop.dfs.TestDataNodeMetrics
  • org.apache.hadoop.dfs.TestDFSPermission
  • org.apache.hadoop.dfs.TestDFSShellGenericOptions
  • org.apache.hadoop.dfs.TestDiskError
  • org.apache.hadoop.dfs.TestFsck
  • org.apache.hadoop.dfs.TestNodeCount
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.