Examples of DFSck

org.apache.hadoop.hdfs.tools.DFSck
This class provides rudimentary checking of DFS volumes for errors and sub-optimal conditions.
The tool scans all files and directories, starting from an indicated root path. The following abnormal conditions are detected and handled:
- files with blocks that are completely missing from all datanodes.
  In this case the tool can perform one of the following actions:
  - none ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_NONE})
  - move corrupted files to /lost+found directory on DFS ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_MOVE}). Remaining data blocks are saved as a block chains, representing longest consecutive series of valid blocks.
  - delete corrupted files ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_DELETE})
- detect files with under-replicated or over-replicated blocks
Additionally, the tool collects a detailed overall DFS statistics, and optionally can print detailed statistics on block locations and replication factors of each file. The tool also provides and option to filter open files during the scan.

Examples of org.apache.hadoop.hdfs.tools.DFSck

  }


  static void runFsck(Configuration conf) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out),
        new String[]{"/", "-files"});
    String result = bStream.toString();
    System.out.println("output from fsck:\n" + result);
    Assert.assertEquals(0, errCode);
    assertTrue(result.contains("/test1"));

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

      // Failover the current standby to active.
      cluster.shutdownNameNode(0);
      cluster.transitionToActive(1);
      
      // Check the FSCK doesn't detect any bad blocks on the SBN.
      int rc = ToolRunner.run(new DFSck(cluster.getConfiguration(1)),
          new String[] { "/", "-files", "-blocks" });
      assertEquals(0, rc);
      
      assertEquals("CorruptBlocks should be empty.", 0, cluster.getNameNode(1)
          .getNamesystem().getCorruptReplicaBlocks());

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

  }


  static void runFsck(Configuration conf) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out),
        new String[]{"/", "-files"});
    String result = bStream.toString();
    System.out.println("output from fsck:\n" + result);
    assertEquals(0, errCode);
    assertTrue(result.contains("/test1"));

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode) {
      assertEquals(expectedErrCode, errCode);
    }
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    FSImage.LOG.error("OUTPUT = " + bStream.toString());

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck


  static String runFsck(Configuration conf, int expectedErrCode,
      boolean checkErrorCode, String... path) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      Assert.assertEquals(expectedErrCode, errCode);
    return bStream.toString();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck


  static String runFsck(Configuration conf, int expectedErrCode,
      boolean checkErrorCode, String... path) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      Assert.assertEquals(expectedErrCode, errCode);
    return bStream.toString();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      assertEquals(expectedErrCode, errCode);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    return bStream.toString();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

    PrintStream oldOut = System.out;
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream newOut = new PrintStream(bStream, true);
    System.setOut(newOut);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf), path);
    if (checkErrorCode)
      assertEquals(expectedErrCode, errCode);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    System.setOut(oldOut);
    return bStream.toString();

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      assertEquals(expectedErrCode, errCode);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    return bStream.toString();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.DFSck

  }
  
  public static void info(String src) throws IOException {
    Configuration conf = getConf(null); 


    DFSck dfsCk = new DFSck(conf);
    dfsCk.run(new String[] { src });
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.