Examples of org.apache.hadoop.hbase.io.FileLink

org.apache.hadoop.hbase.io.FileLink
The FileLink is a sort of hardlink, that allows to access a file given a set of locations.
The Problem:
- HDFS doesn't have support for hardlinks, and this make impossible to referencing the same data blocks using different names.
- HBase store files in one location (e.g. table/region/family/) and when the file is not needed anymore (e.g. compaction, region deletetion, ...) moves it to an archive directory.
If we want to create a reference to a file, we need to remember that it can be in its original location or in the archive folder. The FileLink class tries to abstract this concept and given a set of locations it is able to switch between them making this operation transparent for the user. More concrete implementations of the FileLink are the {@link HFileLink} and the {@link HLogLink}.
Back-references: To help the {@link CleanerChore} to keep track of the links to a particular file,during the FileLink creation, a new file is placed inside a back-reference directory. There's one back-reference directory for each file that has links, and in the directory there's one file per link.
HFileLink Example
- /hbase/table/region-x/cf/file-k (Original File)
- /hbase/table-cloned/region-y/cf/file-k.region-x.table (HFileLink to the original file)
- /hbase/table-2nd-cloned/region-z/cf/file-k.region-x.table (HFileLink to the original file)
- /hbase/.archive/table/region-x/.links-file-k/region-y.table-cloned (Back-reference to the link in table-cloned)
- /hbase/.archive/table/region-x/.links-file-k/region-z.table-cloned (Back-reference to the link in table-2nd-cloned)


    List<Path> files = new ArrayList<Path>();
    files.add(originalPath);
    files.add(archivedPath);


    FileLink link = new FileLink(files);
    FSDataInputStream in = link.open(fs);
    try {
      byte[] data = new byte[8192];
      long size = 0;


      // Read from origin

View Full Code Here

        Path path = new Path(String.format("test-data-%d", i));
        writeSomeData(fs, path, 1 << 20, (byte)i);
        files.add(path);
      }


      FileLink link = new FileLink(files);
      FSDataInputStream in = link.open(fs);
      try {
        byte[] data = new byte[8192];
        int n;


        // Switch to file 1

View Full Code Here

    HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink);
    assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));


    HFileV1Detector t = new HFileV1Detector();
    t.setConf(TEST_UTIL.getConfiguration());
    FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
    //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp)  to look.
    assertTrue(fileLink.getLocations().length == 6);
    for (Path p : fileLink.getLocations()) {
      if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
      if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
      if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
    }
    assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);

View Full Code Here

     * if the file is not found.
     */
    private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            String serverName = fileInfo.getWalServer();
            String logName = fileInfo.getWalName();
            link = new HLogLink(inputRoot, serverName, logName);
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.open(inputFs);
      } catch (IOException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to open source file=" + fileInfo.toString(), e);
        throw e;
      }

View Full Code Here

    }


    private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.getFileStatus(inputFs);
      } catch (FileNotFoundException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
        throw e;
      } catch (IOException e) {

View Full Code Here

     * if the file is not found.
     */
    private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            String serverName = fileInfo.getWalServer();
            String logName = fileInfo.getWalName();
            link = new HLogLink(inputRoot, serverName, logName);
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.open(inputFs);
      } catch (IOException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to open source file=" + fileInfo.toString(), e);
        throw e;
      }

View Full Code Here

    }


    private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.getFileStatus(inputFs);
      } catch (FileNotFoundException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
        throw e;
      } catch (IOException e) {

View Full Code Here

     * if the file is not found.
     */
    private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            String serverName = fileInfo.getWalServer();
            String logName = fileInfo.getWalName();
            link = new HLogLink(inputRoot, serverName, logName);
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.open(inputFs);
      } catch (IOException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to open source file=" + fileInfo.toString(), e);
        throw e;
      }

View Full Code Here

    }


    private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
        throws IOException {
      try {
        FileLink link = null;
        switch (fileInfo.getType()) {
          case HFILE:
            Path inputPath = new Path(fileInfo.getHfile());
            link = new HFileLink(inputRoot, inputArchive, inputPath);
            break;
          case WAL:
            link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
            break;
          default:
            throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
        }
        return link.getFileStatus(inputFs);
      } catch (FileNotFoundException e) {
        context.getCounter(Counter.MISSING_FILES).increment(1);
        LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
        throw e;
      } catch (IOException e) {

View Full Code Here

            try {
              // check whether this path is a reference.
              if (StoreFileInfo.isReference(storeFilePath)) continue;
              // check whether this path is a HFileLink.
              else if (HFileLink.isHFileLink(storeFilePath)) {
                FileLink fLink = getFileLinkWithPreNSPath(storeFilePath);
                fsdis = fLink.open(fs);
                lenToRead = fLink.getFileStatus(fs).getLen();
              } else {
                // a regular hfile
                fsdis = fs.open(storeFilePath);
                lenToRead = storeFile.getLen();
              }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hbase.io.FileLink

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.hbase.io.TestFileLink

org.apache.hadoop.hbase.migration.TestUpgradeTo96

org.apache.hadoop.hbase.snapshot.ExportSnapshot$ExportMapper

org.apache.hadoop.hbase.util.HFileV1Detector

java.io.IOException

java.io.FileNotFoundException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.