Examples of org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes()

Class org.apache.hadoop.hdfs.MiniDFSCluster

Examples of org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes()

org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes()
Modify the config and start up additional DataNodes. The info port for DataNodes is guaranteed to use a free port. Data nodes can run with the name node in the mini cluster or a real name node. For example, running with a real name node is useful when running simulated data nodes with a real name node. If minicluster's name node is null assume that the conf has been set with the right address:port of the name node. @param conf the base configuration to use in starting the DataNodes. Thiswill be modified as necessary. @param numDataNodes Number of DataNodes to start; may be zero @param manageDfsDirs if true, the data directories for DataNodes will becreated and dfs.data.dir will be set in the conf @param operation the operation with which to start the DataNodes. If nullor StartupOption.FORMAT, then StartupOption.REGULAR will be used. @param racks array of strings indicating the rack that each DataNode is on @param hosts array of strings indicating the hostnames for each DataNode @param simulatedCapacities array of capacities of the simulated data nodes @throws IllegalStateException if NameNode has been shutdown

      int neededReplicationSize = 
                           namesystem.blockManager.neededReplications.size();
      
      //Add a new datanode on a different rack
      String newRacks[] = {"/rack2"} ;
      cluster.startDataNodes(conf, 1, true, null, newRacks);


      while ( (numRacks < 2) || (curReplicas < REPLICATION_FACTOR) ||
              (neededReplicationSize > 0) ) {
        LOG.info("Waiting for replication");
        Thread.sleep(600);

View Full Code Here

      int neededReplicationSize = 
                           namesystem.blockManager.neededReplications.size();
      
      //Add a new datanode on a different rack
      String newRacks[] = {"/rack2","/rack2","/rack2"} ;
      cluster.startDataNodes(conf, 3, true, null, newRacks);
      REPLICATION_FACTOR = 5;
      namesystem.setReplication(FILE_NAME, REPLICATION_FACTOR); 


      while ( (numRacks < 2) || (curReplicas < REPLICATION_FACTOR) ||
              (neededReplicationSize > 0) ) {

View Full Code Here

      // expect the result is corrupt
      assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
      System.out.println(outStr);
      
      // bring up data nodes & cleanup cluster
      cluster.startDataNodes(conf, 4, true, null, null);
      cluster.waitActive();
      cluster.waitClusterUp();
      fs = cluster.getFileSystem();
      util.cleanup(fs, "/srcdat");
    } finally {

View Full Code Here

      assertEquals(file5.getName(), fileSplit.getPath(1).getName());
      assertEquals(0, fileSplit.getOffset(1));
      assertEquals(BLOCKSIZE, fileSplit.getLength(1));
      assertEquals(hosts1[0], fileSplit.getLocations()[0]);
      
      dfs.startDataNodes(conf, 1, true, null, rack2, hosts2, null);
      dfs.waitActive();


      // create file on two datanodes.
      Path file2 = new Path(dir2 + "/file2");
      writeFile(conf, file2, (short)2, 2);

View Full Code Here

      assertEquals(0, fileSplit.getOffset(0));
      assertEquals(BLOCKSIZE, fileSplit.getLength(0));
      assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1


      // create another file on 3 datanodes and 3 racks.
      dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null);
      dfs.waitActive();
      Path file3 = new Path(dir3 + "/file3");
      writeFile(conf, new Path(dir3 + "/file3"), (short)3, 3);
      inFormat = new DummyInputFormat();
      FileInputFormat.setInputPaths(job, dir1 + "," + dir2 + "," + dir3);

View Full Code Here

      assertEquals(file5.getName(), fileSplit.getPath(1).getName());
      assertEquals(0, fileSplit.getOffset(1));
      assertEquals(f5.getLen(), fileSplit.getLength(1));
      assertEquals(hosts1[0], fileSplit.getLocations()[0]);
      
      dfs.startDataNodes(conf, 1, true, null, rack2, hosts2, null);
      dfs.waitActive();


      // create file on two datanodes.
      Path file2 = new Path(dir2 + "/file2.gz");
      FileStatus f2 = writeGzipFile(conf, file2, (short)2, 2);

View Full Code Here

      assertEquals(0, fileSplit.getOffset(0));
      assertEquals(f1.getLen(), fileSplit.getLength(0));
      assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1


      // create another file on 3 datanodes and 3 racks.
      dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null);
      dfs.waitActive();
      Path file3 = new Path(dir3 + "/file3.gz");
      FileStatus f3 = writeGzipFile(conf, file3, (short)3, 3);
      inFormat = new DummyInputFormat();
      FileInputFormat.setInputPaths(job, dir1 + "," + dir2 + "," + dir3);

View Full Code Here

      FileSystem fs = cluster.getFileSystem();
      Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
      out = fs.create(testPath, (short) 2);
      out.writeBytes("HDFS-3157: " + testPath);
      out.hsync();
      cluster.startDataNodes(conf, 1, true, null, null, null);
      String bpid = namesystem.getBlockPoolId();
      ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
      Block block = blk.getLocalBlock();
      DataNode dn = cluster.getDataNodes().get(0);

View Full Code Here

      conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "bnCluster");
      conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".bnCluster",
          "nnActive, nnBackup");
      conf.set(rpcAddrKeyPreffix + ".nnActive", nnAddr);
      conf.set(rpcAddrKeyPreffix + ".nnBackup", bnAddr);
      cluster.startDataNodes(conf, 3, true, StartupOption.REGULAR, null);


      DFSTestUtil.createFile(
          fileSys, file1, 8192, (short)3, 0);


      // Read the same file from file systems pointing to NN and BN

View Full Code Here

                fs.exists(restartFile));
    assertFalse("Temporary restart-file exists after update failure", 
                fs.exists(restartFile));


    // start 1 data node
    dfs.startDataNodes(conf, 1, true, null, null, null, null);
    dfs.waitActive();


    failed = false;
    try {
      jobtracker.recoveryManager.updateRestartCount();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.