Examples of org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker

Package org.apache.hadoop.hbase.util.hbck

Examples of org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker

org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker
This class marches through all of the region's hfiles and verifies that they are all valid files. One just needs to instantiate the class, use checkTables(List) and then retrieve the corrupted hfiles (and quarantined files if in quarantining mode) The implementation currently parallelizes at the regionDir level.

  public void setSidelineDir(String sidelineDir) {
    this.sidelineDir = new Path(sidelineDir);
  }


  protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
  }

View Full Code Here

    connect();


    // if corrupt file mode is on, first fix them since they may be opened later
    if (checkCorruptHFiles || sidelineCorruptHFiles) {
      LOG.info("Checking all hfiles for corruption");
      HFileCorruptionChecker hfcc = createHFileCorruptionChecker(sidelineCorruptHFiles);
      setHFileCorruptionChecker(hfcc); // so we can get result
      Collection<String> tables = getIncludedTables();
      Collection<Path> tableDirs = new ArrayList<Path>();
      Path rootdir = FSUtils.getRootDir(getConf());
      if (tables.size() > 0) {
        for (String t : tables) {
          tableDirs.add(FSUtils.getTablePath(rootdir, t));
        }
      } else {
        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
      }
      hfcc.checkTables(tableDirs);
      hfcc.report(errors);
    }


    // check and fix table integrity, region consistency.
    int code = onlineHbck();
    setRetCode(code);

View Full Code Here

      HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));


      // we cannot enable here because enable never finished due to the corrupt region.
      HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
      assertEquals(res.getRetCode(), 0);
      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
      assertEquals(hfcc.getHFilesChecked(), 5);
      assertEquals(hfcc.getCorrupted().size(), 1);
      assertEquals(hfcc.getFailures().size(), 0);
      assertEquals(hfcc.getQuarantined().size(), 1);
      assertEquals(hfcc.getMissing().size(), 0);


      // Its been fixed, verify that we can enable.
      TEST_UTIL.getHBaseAdmin().enableTable(table);
    } finally {
      deleteTable(table);

View Full Code Here

      
      String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission", table};
      ExecutorService exec = new ScheduledThreadPoolExecutor(10);
      HBaseFsck res = hbck.exec(exec, args);


      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
      assertEquals(hfcc.getHFilesChecked(), check);
      assertEquals(hfcc.getCorrupted().size(), corrupt);
      assertEquals(hfcc.getFailures().size(), fail);
      assertEquals(hfcc.getQuarantined().size(), quar);
      assertEquals(hfcc.getMissing().size(), missing);


      // its been fixed, verify that we can enable
      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
      admin.enableTableAsync(table);
      while (!admin.isTableEnabled(table)) {

View Full Code Here

    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
    // inject a fault in the hfcc created.
    final FileSystem fs = FileSystem.get(conf);
    HBaseFsck hbck = new HBaseFsck(conf, exec) {
      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
          boolean attemptedFirstHFile = false;
          protected void checkHFile(Path p) throws IOException {
            if (!attemptedFirstHFile) {
              attemptedFirstHFile = true;
              assertTrue(fs.delete(p, true)); // make sure delete happened.

View Full Code Here

    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
    // inject a fault in the hfcc created.
    final FileSystem fs = FileSystem.get(conf);
    HBaseFsck hbck = new HBaseFsck(conf, exec) {
      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
          boolean attemptedFirstFamDir = false;
          protected void checkColFamDir(Path p) throws IOException {
            if (!attemptedFirstFamDir) {
              attemptedFirstFamDir = true;
              assertTrue(fs.delete(p, true)); // make sure delete happened.

View Full Code Here

    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
    // inject a fault in the hfcc created.
    final FileSystem fs = FileSystem.get(conf);
    HBaseFsck hbck = new HBaseFsck(conf, exec) {
      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
          boolean attemptedFirstRegionDir = false;
          protected void checkRegionDir(Path p) throws IOException {
            if (!attemptedFirstRegionDir) {
              attemptedFirstRegionDir = true;
              assertTrue(fs.delete(p, true)); // make sure delete happened.

View Full Code Here

      HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));


      // we cannot enable here because enable never finished due to the corrupt region.
      HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
      assertEquals(res.getRetCode(), 0);
      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
      assertEquals(hfcc.getHFilesChecked(), 5);
      assertEquals(hfcc.getCorrupted().size(), 1);
      assertEquals(hfcc.getFailures().size(), 0);
      assertEquals(hfcc.getQuarantined().size(), 1);
      assertEquals(hfcc.getMissing().size(), 0);


      // Its been fixed, verify that we can enable.
      TEST_UTIL.getHBaseAdmin().enableTable(table);
    } finally {
      deleteTable(table);

View Full Code Here

      String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
          table.getNameAsString()};
      ExecutorService exec = new ScheduledThreadPoolExecutor(10);
      HBaseFsck res = hbck.exec(exec, args);


      HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
      assertEquals(hfcc.getHFilesChecked(), check);
      assertEquals(hfcc.getCorrupted().size(), corrupt);
      assertEquals(hfcc.getFailures().size(), fail);
      assertEquals(hfcc.getQuarantined().size(), quar);
      assertEquals(hfcc.getMissing().size(), missing);


      // its been fixed, verify that we can enable
      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
      admin.enableTableAsync(table);
      while (!admin.isTableEnabled(table)) {

View Full Code Here

    // inject a fault in the hfcc created.
    final FileSystem fs = FileSystem.get(conf);
    HBaseFsck hbck = new HBaseFsck(conf, exec) {
      @Override
      public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
        return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
          boolean attemptedFirstHFile = false;
          @Override
          protected void checkHFile(Path p) throws IOException {
            if (!attemptedFirstHFile) {
              attemptedFirstHFile = true;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker

org.apache.hadoop.fs.Path

org.apache.hadoop.hbase.io.hfile.CacheConfig

org.apache.hadoop.hbase.util.FSUtils.FamilyDirFilter

org.apache.hadoop.hbase.util.FSUtils.HFileFilter

org.apache.hadoop.hbase.util.FSUtils.RegionDirFilter

org.apache.hadoop.hbase.util.HBaseFsck

org.apache.hadoop.hbase.util.TestHBaseFsck

org.apache.hadoop.hbase.util.TestHBaseFsckEncryption

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.