Examples of org.apache.hadoop.hdfs.tools.DFSck

org.apache.hadoop.hdfs.tools.DFSck
This class provides rudimentary checking of DFS volumes for errors and sub-optimal conditions.
The tool scans all files and directories, starting from an indicated root path. The following abnormal conditions are detected and handled:
- files with blocks that are completely missing from all datanodes.
  In this case the tool can perform one of the following actions:
  - none ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_NONE})
  - move corrupted files to /lost+found directory on DFS ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_MOVE}). Remaining data blocks are saved as a block chains, representing longest consecutive series of valid blocks.
  - delete corrupted files ( {@link org.apache.hadoop.hdfs.server.namenode.NamenodeFsck#FIXING_DELETE})
- detect files with under-replicated or over-replicated blocks
Additionally, the tool collects a detailed overall DFS statistics, and optionally can print detailed statistics on block locations and replication factors of each file. The tool also provides and option to filter open files during the scan.

    PrintStream oldOut = System.out;
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream newOut = new PrintStream(bStream, true);
    System.setOut(newOut);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf), path);
    if (checkErrorCode)
      assertEquals(expectedErrCode, errCode);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    System.setOut(oldOut);
    return bStream.toString();

View Full Code Here

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode) {
      assertEquals(expectedErrCode, errCode);
    }
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    FSImage.LOG.error("OUTPUT = " + bStream.toString());

View Full Code Here


  static String runFsck(Configuration conf, int expectedErrCode,
      boolean checkErrorCode, String... path) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      Assert.assertEquals(expectedErrCode, errCode);
    return bStream.toString();
  }

View Full Code Here

    PrintStream oldOut = System.out;
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream newOut = new PrintStream(bStream, true);
    System.setOut(newOut);
    ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf), path);
    if (checkErrorCode)
      assertEquals(expectedErrCode, errCode);
    ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    System.setOut(oldOut);
    return bStream.toString();

View Full Code Here

  private BufferedReader getLostFileReader(String[] dfsckArgs) 
      throws IOException {


    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(bout, true);
    DFSck dfsck = new DFSck(getConf(), ps);
    try {
      dfsck.run(dfsckArgs);
    } catch (Exception e) {
      throw new IOException(e);
    }
    ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
    return new BufferedReader(new InputStreamReader(bin));

View Full Code Here


  public static String[] getCorruptFiles(Configuration conf)
    throws IOException {
    ByteArrayOutputStream baseOut = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(baseOut, true);
    DFSck fsck = new DFSck(conf, out);
    String[] args = new String[]{"-list-corruptfileblocks"};
    try {
      ToolRunner.run(fsck, args);
    } catch (Exception e) {
      throw new IOException("DFSck.run exception ", e);

View Full Code Here

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode) {
      assertEquals(expectedErrCode, errCode);
    }
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    return bStream.toString();

View Full Code Here

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode) {
      assertEquals(expectedErrCode, errCode);
    }
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    FSImage.LOG.error("OUTPUT = " + bStream.toString());

View Full Code Here


  static String runFsck(Configuration conf, int expectedErrCode,
      boolean checkErrorCode, String... path) throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode)
      Assert.assertEquals(expectedErrCode, errCode);
    return bStream.toString();
  }

View Full Code Here

                        boolean checkErrorCode,String... path) 
                        throws Exception {
    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
    PrintStream out = new PrintStream(bStream, true);
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.ALL);
    int errCode = ToolRunner.run(new DFSck(conf, out), path);
    if (checkErrorCode) {
      assertEquals(expectedErrCode, errCode);
    }
    ((Log4JLogger)FSPermissionChecker.LOG).getLogger().setLevel(Level.INFO);
    FSImage.LOG.error("OUTPUT = " + bStream.toString());

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.hdfs.tools.DFSck

com.nexr.rhive.hadoop.FSUtils

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.hdfs.DistributedFileSystem

org.apache.hadoop.hdfs.HdfsConfiguration

org.apache.hadoop.hdfs.RaidDFSUtil

org.apache.hadoop.hdfs.server.namenode.ha.TestHAAppend

org.apache.hadoop.hdfs.server.namenode.ha.TestHAFsck

org.apache.hadoop.hdfs.server.namenode.TestFsck

org.apache.hadoop.hdfs.TestClientReportBadBlock

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.