Examples of org.apache.hadoop.hbase.io.Reference

org.apache.hadoop.hbase.io.Reference
A reference to the top or bottom half of a store file where 'bottom' is the first half of the file containing the keys that sort lowest and 'top' is the second half of the file with keys that sort greater than those of the bottom half. The file referenced lives under a different region. References are made at region split time.
References work with a special half store file type. References know how to write out the reference format in the file system and are what is juggled when references are mixed in with direct store files. The half store file type is used reading the referred to file.
References to store files located over in some other region look like this in the file system 1278437856009925445.3323223323: i.e. an id followed by hash of the referenced region. Note, a region is itself not splittable if it has instances of store file references. References are cleaned up by compactions.

   * filters, etc.
   */
  static void splitStoreFile(Configuration conf, Path inFile, HColumnDescriptor familyDesc,
      byte[] splitKey, Path bottomOut, Path topOut) throws IOException {
    // Open reader with no block cache, and not in-memory
    Reference topReference = new Reference(splitKey, Range.top);
    Reference bottomReference = new Reference(splitKey, Range.bottom);


    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
  }

View Full Code Here

      }


      // Split each store file.
      for(HStoreFile h: hstoreFilesToSplit) {
        // A reference to the bottom half of the hsf store file.
        Reference aReference = new Reference(
            this.regionInfo.getEncodedName(), h.getFileId(),
            new HStoreKey(midKey, this.regionInfo), Reference.Range.bottom);
        HStoreFile a = new HStoreFile(this.conf, fs, splits,
            regionAInfo, h.getColFamily(), -1, aReference);
        // Reference to top half of the hsf store file.
        Reference bReference = new Reference(
            this.regionInfo.getEncodedName(), h.getFileId(),
            new HStoreKey(midKey, this.regionInfo), Reference.Range.top);
        HStoreFile b = new HStoreFile(this.conf, fs, splits,
            regionBInfo, h.getColFamily(), -1, bReference);
        h.splitStoreFile(a, b, this.fs);

View Full Code Here


    f.getReader().close(true);


    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r =
      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfoForFs.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      final StoreFile f, final Path mergedDir)
      throws IOException {
    Path referenceDir = new Path(new Path(mergedDir,
        mergedRegion.getEncodedName()), familyName);
    // A whole reference to the store file.
    Reference r = Reference.createTopReference(regionInfoForFs.getStartKey());
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above. The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String mergingRegionName = regionInfoForFs.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(referenceDir, f.getPath().getName() + "."
        + mergingRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      Path rootdir = services.getMasterFileSystem().getRootDir();
      Path tabledir =
        FSUtils.getTableDir(rootdir, htd.getTableName());
      Path storedir = HStore.getStoreHomedir(tabledir, splita,
          htd.getColumnFamilies()[0].getName());
      Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
      long now = System.currentTimeMillis();
      // Reference name has this format: StoreFile#REF_NAME_PARSER
      Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
      FileSystem fs = services.getMasterFileSystem().getFileSystem();
      Path path = ref.write(fs, p);
      assertTrue(fs.exists(path));
      assertFalse(janitor.cleanParent(parent, r));
      // Remove the reference file and try again.
      assertTrue(fs.delete(p, true));
      assertTrue(janitor.cleanParent(parent, r));

View Full Code Here

  throws IOException {
    Path rootdir = services.getMasterFileSystem().getRootDir();
    Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
    Path storedir = HStore.getStoreHomedir(tabledir, daughter,
      htd.getColumnFamilies()[0].getName());
    Reference ref =
      top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
    long now = System.currentTimeMillis();
    // Reference name has this format: StoreFile#REF_NAME_PARSER
    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
    FileSystem fs = services.getMasterFileSystem().getFileSystem();
    ref.write(fs, p);
    return p;
  }

View Full Code Here


    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);


    // Create the new reference
    if (storeFile.hasReference()) {
      Reference reference = Reference.convert(storeFile.getReference());
      reference.write(fs, outPath);
    } else {
      InputStream in;
      if (linkPath != null) {
        in = new HFileLink(conf, linkPath).open(fs);
      } else {

View Full Code Here


    f.getReader().close(true);


    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r =
      top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfo.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      final StoreFile f, final Path mergedDir)
      throws IOException {
    Path referenceDir = new Path(new Path(mergedDir,
        mergedRegion.getEncodedName()), familyName);
    // A whole reference to the store file.
    Reference r = Reference.createTopReference(regionInfo.getStartKey());
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above. The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String mergingRegionName = regionInfo.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(referenceDir, f.getPath().getName() + "."
        + mergingRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      Configuration conf, Path inFile,
      HColumnDescriptor familyDesc, byte[] splitKey,
      Path bottomOut, Path topOut) throws IOException
  {
    // Open reader with no block cache, and not in-memory
    Reference topReference = Reference.createTopReference(splitKey);
    Reference bottomReference = Reference.createBottomReference(splitKey);


    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hbase.io.Reference

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.hbase.HStoreKey

org.apache.hadoop.hbase.index.io.TestIndexHalfStoreFileReader

org.apache.hadoop.hbase.index.mapreduce.IndexLoadIncrementalHFile

org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles

org.apache.hadoop.hbase.master.TestCatalogJanitor

org.apache.hadoop.hbase.regionserver.HRegion

org.apache.hadoop.hbase.regionserver.HRegionFileSystem

org.apache.hadoop.hbase.regionserver.HStore

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.