Examples of org.apache.hadoop.hbase.regionserver.HRegionFileSystem

Package org.apache.hadoop.hbase.regionserver

Examples of org.apache.hadoop.hbase.regionserver.HRegionFileSystem

org.apache.hadoop.hbase.regionserver.HRegionFileSystem
Acts as an abstraction layer b/w HBase and underlying fs. Used for making non-idempotent calls. This is useful as it can have a retry logic for such operations, as they are not retried at hdfs level. Region specific methods that access fs should be added here.

    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
    Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);


    // First region, simple with one plain hfile.
    HRegionInfo hri = new HRegionInfo(tableWithRefsName);
    HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
      fs, FSUtils.getTableDir(archiveDir, hri.getTableName()), hri);
    Path storeFile = new Path(rootDir, TEST_HFILE);
    FSDataOutputStream out = fs.create(storeFile);
    out.write(Bytes.toBytes("Test Data"));
    out.close();
    r0fs.commitStoreFile(TEST_FAMILY, storeFile);


    // Second region, used to test the split case.
    // This region contains a reference to the hfile in the first region.
    hri = new HRegionInfo(tableWithRefsName);
    HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
      fs, new Path(archiveDir, hri.getTableName().getNameAsString()), hri);
    storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
    out = fs.create(storeFile);
    out.write(Bytes.toBytes("Test Data"));
    out.close();
    r1fs.commitStoreFile(TEST_FAMILY, storeFile);


    Path tableDir = FSUtils.getTableDir(archiveDir, tableWithRefsName);
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
    SnapshotDescriptionUtils.writeSnapshotInfo(sd, snapshotDir, fs);

View Full Code Here

          byte[] sk = hri.getStartKey();
          if (sk.length == 0)
            sk = splitAlgo.firstRow();
          String startKey = splitAlgo.rowToStr(sk);


          HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
              table.getConfiguration(), fs, tableDir, hri, true);


          // check every Column Family for that region
          boolean refFound = false;
          for (HColumnDescriptor c : htd.getFamilies()) {
            if ((refFound = regionFs.hasReferences(htd.getNameAsString()))) {
              break;
            }
          }


          // compaction is completed when all reference files are gone

View Full Code Here

      // 2. for each region, write all the info to disk
      LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
          + ClientSnapshotDescriptionUtils.toString(snapshot));
      for (HRegionInfo regionInfo : regions) {
        // 2.1 copy the regionInfo files to the snapshot
        HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
          snapshotDir, regionInfo);


        // check for error for each region
        monitor.rethrowException();


        // 2.2 for each region, copy over its recovered.edits directory
        Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
        Path snapshotRegionDir = regionFs.getRegionDir();
        new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
        monitor.rethrowException();


        // 2.3 reference all the files in the region
        new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();

View Full Code Here

    Path rootdir = this.services.getMasterFileSystem().getRootDir();
    Path tabledir = HTableDescriptor.getTableDir(rootdir,
        mergedRegion.getTableName());
    HTableDescriptor htd = getTableDescriptor(mergedRegion
        .getTableNameAsString());
    HRegionFileSystem regionFs = null;
    try {
      regionFs = HRegionFileSystem.openRegionFromFileSystem(
          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
    } catch (IOException e) {
      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
    }
    if (regionFs == null || !regionFs.hasReferences(htd)) {
      LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
          + regionB.getRegionNameAsString()
          + " from fs because merged region no longer holds references");
      HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
      HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);

View Full Code Here


    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
    Path rootdir = this.services.getMasterFileSystem().getRootDir();
    Path tabledir = HTableDescriptor.getTableDir(rootdir, daughter.getTableName());


    HRegionFileSystem regionFs = null;
    try {
      regionFs = HRegionFileSystem.openRegionFromFileSystem(
          this.services.getConfiguration(), fs, tabledir, daughter, true);
    } catch (IOException e) {
      LOG.warn("Daughter region does not exist: " + daughter.getEncodedName());
      return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
    }


    boolean references = false;
    HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTableNameAsString());
    for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
      if ((references = regionFs.hasReferences(family.getNameAsString()))) {
        break;
      }
    }
    return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
  }

View Full Code Here

     * the store dir to compact as source.
     */
    private static HStore getStore(final Configuration conf, final FileSystem fs,
        final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri,
        final String familyName, final Path tempDir) throws IOException {
      HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) {
        @Override
        public Path getTempDir() {
          return tempDir;
        }
      };

View Full Code Here

     * the store dir to compact as source.
     */
    private static HStore getStore(final Configuration conf, final FileSystem fs,
        final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri,
        final String familyName, final Path tempDir) throws IOException {
      HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, hri) {
        @Override
        public Path getTempDir() {
          return tempDir;
        }
      };

View Full Code Here

    fs.delete(new Path(snapshotDir, regionName), true);
  }


  static SnapshotRegionManifest buildManifestFromDisk (final Configuration conf,
      final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
          tableDir, regionInfo, true);
    SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();


    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    manifest.setRegionInfo(HRegionInfo.convert(regionInfo));


    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");


    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
    // batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently 1000
    // files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
      for (String familyName: familyNames) {
        Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
        if (storeFiles == null) {
          LOG.debug("No files under family: " + familyName);
          continue;
        }

View Full Code Here

      this.conf = conf;
      this.fs = fs;
    }


    public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
      HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
        fs, snapshotDir, regionInfo);
      return snapshotRegionFs;
    }

View Full Code Here

          byte[] sk = hri.getStartKey();
          if (sk.length == 0)
            sk = splitAlgo.firstRow();
          String startKey = splitAlgo.rowToStr(sk);


          HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
              table.getConfiguration(), fs, tableDir, hri, true);


          // check every Column Family for that region
          boolean refFound = false;
          for (HColumnDescriptor c : htd.getFamilies()) {
            if ((refFound = regionFs.hasReferences(htd.getTableName().getNameAsString()))) {
              break;
            }
          }


          // compaction is completed when all reference files are gone

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.HRegionFileSystem

co.cask.tephra.hbase96.coprocessor.TransactionProcessorTest

co.cask.tephra.hbase98.coprocessor.TransactionProcessorTest

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.PathFilter

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.hbase.io.Reference

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.