Examples of createJobConf()

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

102
103
104
105
106
107
108
109
110
111
112
      writeASinkFile(conf, fileSys, DEMUX_INPUT_PATH, LINES);

      System.out.println("wrote " +
      fileSys.getFileStatus(DEMUX_INPUT_PATH).getLen() + " bytes of temp test data");
      long ts_start = System.currentTimeMillis();
      runDemux(mr.createJobConf(), DEMUX_INPUT_PATH, DEMUX_OUTPUT_PATH);

      long time = (System.currentTimeMillis() - ts_start);
      long bytes = fileSys.getContentSummary(DEMUX_OUTPUT_PATH).getLength();
      System.out.println("result was " + bytes + " bytes long");
      System.out.println("processing took " + time + " milliseconds");
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

71
72
73
74
75
76
77
78
79
80
81
      dfs = new MiniDFSCluster(conf, 1, true, null);
      FileSystem fileSys = dfs.getFileSystem();
      String namenode = fileSys.getUri().toString();

      mr  = new MiniMRCluster(1, namenode, 3);
      String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
      String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name");
      String argv[] = new String[] {
        "-input", INPUT_FILE,
        "-output", OUTPUT_DIR,
        "-mapper", map,
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

72
73
74
75
76
77
78
79
80
81
82
      FileSystem fileSys = dfs.getFileSystem();
      String namenode = fileSys.getUri().toString();

      mr  = new MiniMRCluster(1, namenode, 3);
      String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
      String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name");
      String argv[] = new String[] {
        "-input", INPUT_FILE,
        "-output", OUTPUT_DIR,
        "-mapper", map,
        "-reducer", reduce,
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

749
750
751
752
753
754
755
756
757
758
759
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCpV1(job),
          new String[] {"-m", "100",
                        "-log",
                        namenode+"/logs",
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

62
63
64
65
66
67
68
69
70
71
72
      FileSystem fileSys = dfs.getFileSystem();
      String namenode = fileSys.getUri().toString();
      mr  = new MiniMRCluster(1, namenode, 3);
      // During tests, the default Configuration will use a local mapred
      // So don't specify -config or -cluster
      String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
      String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name");
      String argv[] = new String[] {
        "-input", INPUT_FILE,
        "-output", OUTPUT_DIR,
        "-mapper", map,
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

63
64
65
66
67
68
69
70
71
72
73
      String namenode = fileSys.getUri().toString();
      mr  = new MiniMRCluster(1, namenode, 3);
      // During tests, the default Configuration will use a local mapred
      // So don't specify -config or -cluster
      String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
      String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name");
      String argv[] = new String[] {
        "-input", INPUT_FILE,
        "-output", OUTPUT_DIR,
        "-mapper", map,
        "-reducer", reduce,
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

166
167
168
169
170
171
172
173
174
175
176
      }
      System.out.println("args=" + Arrays.asList(args).toString().replace(",", ",\n  "));
      System.out.println("newstatus=" + Arrays.asList(newstatus).toString().replace(",", ",\n  "));

      //run DistCh
      new DistCh(mr.createJobConf()).run(args);
      runLsr(shell, tree.root, 0);

      //check results
      for(int i = 0; i < NUN_SUBS; i++) {
        Path sub = new Path(tree.root + "/sub" + i);
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

83
84
85
86
87
88
89
90
91
92
93
  public void testWithLocal() throws Exception {
    MiniMRCluster mr = null;
    try {
      mr = new MiniMRCluster(2, "file:///", 3);
      Configuration conf = mr.createJobConf();
      runWordCount(conf);
      runSecondarySort(conf);
    } finally {
      if (mr != null) { mr.shutdown(); }
    }
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

58
59
60
61
62
63
64
65
66
67
68
      JobConf mrConf = new JobConf(conf);
      mr = new MiniMRCluster(slaves, fileSys.getUri().toString(), 1,
                             null, null, mrConf);

      // Run examples
      TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf));
      TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf));
    } finally {
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
View Full Code Here

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

59
60
61
62
63
64
65
66
67
68
69
      mr = new MiniMRCluster(slaves, fileSys.getUri().toString(), 1,
                             null, null, mrConf);

      // Run examples
      TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf));
      TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf));
    } finally {
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.