Examples of org.apache.hadoop.hbase.io.Reference

org.apache.hadoop.hbase.io.Reference
A reference to the top or bottom half of a store file where 'bottom' is the first half of the file containing the keys that sort lowest and 'top' is the second half of the file with keys that sort greater than those of the bottom half. The file referenced lives under a different region. References are made at region split time.
References work with a special half store file type. References know how to write out the reference format in the file system and are what is juggled when references are mixed in with direct store files. The half store file type is used reading the referred to file.
References to store files located over in some other region look like this in the file system 1278437856009925445.3323223323: i.e. an id followed by hash of the referenced region. Note, a region is itself not splittable if it has instances of store file references. References are cleaned up by compactions.

                    final StoreFile f,
                    final byte [] splitRow,
                    final Reference.Range range)
      throws IOException {
    // A reference to the bottom half of the hsf store file.
    Reference r = new Reference(splitRow, range);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = f.getPath().getParent().getParent().getName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      Configuration conf, Path inFile,
      HColumnDescriptor familyDesc, byte[] splitKey,
      Path bottomOut, Path topOut) throws IOException
  {
    // Open reader with no block cache, and not in-memory
    Reference topReference = new Reference(splitKey, Range.top);
    Reference bottomReference = new Reference(splitKey, Range.bottom);


    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
  }

View Full Code Here

    HStoreKey midkey = new HStoreKey(((HStoreKey)reader.midKey()).getRow());
    HStoreKey hsk = new HStoreKey();
    reader.finalKey(hsk);
    byte [] finalKey = hsk.getRow();
    // Make a reference for the bottom half of the just written file.
    Reference reference =
      new Reference(hsf.getEncodedRegionName(), hsf.getFileId(),
          midkey, Reference.Range.top);
    HStoreFile refHsf = new HStoreFile(this.conf, this.fs, 
        new Path(DIR, getName()),
        HRegionInfo.FIRST_META_REGIONINFO,
        hsf.getColFamily(), 456, reference);
    // Assert that reference files are written and that we can write and
    // read the info reference file at least.
    refHsf.writeReferenceFiles(this.fs);
    assertTrue(this.fs.exists(refHsf.getMapFilePath()));
    assertTrue(this.fs.exists(refHsf.getInfoFilePath()));
    Reference otherReference =
      HStoreFile.readSplitInfo(refHsf.getInfoFilePath(), this.fs);
    assertEquals(reference.getEncodedRegionName(),
        otherReference.getEncodedRegionName());
    assertEquals(reference.getFileId(),
        otherReference.getFileId());
    assertEquals(reference.getMidkey().toString(),
        otherReference.getMidkey().toString());
    // Now confirm that I can read from the reference and that it only gets
    // keys from top half of the file.
    MapFile.Reader halfReader = refHsf.getReader(this.fs, false, false);
    HStoreKey key = new HStoreKey();
    ImmutableBytesWritable value = new ImmutableBytesWritable();

View Full Code Here

      Configuration conf, Path inFile,
      HColumnDescriptor familyDesc, byte[] splitKey,
      Path bottomOut, Path topOut) throws IOException
  {
    // Open reader with no block cache, and not in-memory
    Reference topReference = Reference.createTopReference(splitKey);
    Reference bottomReference = Reference.createBottomReference(splitKey);


    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
  }

View Full Code Here

        return null;
      }
    }
    
    // A reference to the bottom half of the hsf store file.
    Reference r = new Reference(splitRow, range);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = f.getPath().getParent().getParent().getName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
  }

View Full Code Here

      Configuration conf, Path inFile,
      HColumnDescriptor familyDesc, byte[] splitKey,
      Path bottomOut, Path topOut) throws IOException
  {
    // Open reader with no block cache, and not in-memory
    Reference topReference = new Reference(splitKey, Range.top);
    Reference bottomReference = new Reference(splitKey, Range.bottom);


    copyHFileHalf(conf, inFile, topOut, topReference, familyDesc);
    copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc);
  }

View Full Code Here

      Path rootdir = services.getMasterFileSystem().getRootDir();
      Path tabledir =
        HTableDescriptor.getTableDir(rootdir, htd.getName());
      Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(),
          htd.getColumnFamilies()[0].getName());
      Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top);
      long now = System.currentTimeMillis();
      // Reference name has this format: StoreFile#REF_NAME_PARSER
      Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
      FileSystem fs = services.getMasterFileSystem().getFileSystem();
      Path path = ref.write(fs, p);
      assertTrue(fs.exists(path));
      assertFalse(janitor.cleanParent(parent, r));
      // Remove the reference file and try again.
      assertTrue(fs.delete(p, true));
      assertTrue(janitor.cleanParent(parent, r));

View Full Code Here

  throws IOException {
    Path rootdir = services.getMasterFileSystem().getRootDir();
    Path tabledir = HTableDescriptor.getTableDir(rootdir, parent.getTableName());
    Path storedir = Store.getStoreHomedir(tabledir, daughter.getEncodedName(),
      htd.getColumnFamilies()[0].getName());
    Reference ref = new Reference(midkey,
      top? Reference.Range.top: Reference.Range.bottom);
    long now = System.currentTimeMillis();
    // Reference name has this format: StoreFile#REF_NAME_PARSER
    Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
    FileSystem fs = services.getMasterFileSystem().getFileSystem();
    ref.write(fs, p);
    return p;
  }

View Full Code Here

   */
  static Reference readSplitInfo(final Path p, final FileSystem fs)
  throws IOException {
    FSDataInputStream in = fs.open(p);
    try {
      Reference r = new Reference();
      r.readFields(in);
      return r;
    } finally {
      in.close();
    }
  }

View Full Code Here

      }


      // Split each store file.
      for(HStoreFile h: hstoreFilesToSplit) {
        // A reference to the bottom half of the hsf store file.
        Reference aReference = new Reference(
            this.regionInfo.getEncodedName(), h.getFileId(),
            new HStoreKey(midKey, this.regionInfo), Reference.Range.bottom);
        HStoreFile a = new HStoreFile(this.conf, fs, splits,
            regionAInfo, h.getColFamily(), -1, aReference);
        // Reference to top half of the hsf store file.
        Reference bReference = new Reference(
            this.regionInfo.getEncodedName(), h.getFileId(),
            new HStoreKey(midKey, this.regionInfo), Reference.Range.top);
        HStoreFile b = new HStoreFile(this.conf, fs, splits,
            regionBInfo, h.getColFamily(), -1, bReference);
        h.splitStoreFile(a, b, this.fs);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hbase.io.Reference

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.hbase.HStoreKey

org.apache.hadoop.hbase.index.io.TestIndexHalfStoreFileReader

org.apache.hadoop.hbase.index.mapreduce.IndexLoadIncrementalHFile

org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles

org.apache.hadoop.hbase.master.TestCatalogJanitor

org.apache.hadoop.hbase.regionserver.HRegion

org.apache.hadoop.hbase.regionserver.HRegionFileSystem

org.apache.hadoop.hbase.regionserver.HStore

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.