Examples of org.apache.hadoop.hbase.regionserver.StoreFile$WriterBuilder

Package org.apache.hadoop.hbase.regionserver

Examples of org.apache.hadoop.hbase.regionserver.StoreFile$WriterBuilder

org.apache.hadoop.hbase.regionserver.StoreFile
A Store data file. Stores usually have one or more of these files. They are produced by flushing the memstore to disk. To create, instantiate a writer using {@link StoreFile#WriterBuilder}and append data. Be sure to add any metadata before calling close on the Writer (Use the appendMetadata convenience methods). On close, a StoreFile is sitting in the Filesystem. To refer to it, create a StoreFile instance passing filesystem and path. To read, call {@link #createReader()}.
StoreFiles may also reference store files in another Store. The reason for this weird pattern where you use a different instance for the writer and a reader is that we write once but read a lot more.

    when(si.getLevel0Files()).thenReturn(l0Files);
    return si;
  }


  private static StoreFile createFile(long size) throws Exception {
    StoreFile sf = mock(StoreFile.class);
    when(sf.getPath()).thenReturn(new Path("moo"));
    StoreFile.Reader r = mock(StoreFile.Reader.class);
    when(r.getEntries()).thenReturn(size);
    when(r.length()).thenReturn(size);
    when(sf.getReader()).thenReturn(r);
    return sf;
  }

View Full Code Here

  protected StoreFile createMockStoreFileBytes(final long size) {
    return createMockStoreFile(size, -1L);
  }


  protected StoreFile createMockStoreFile(final long sizeInBytes, final long seqId) {
    StoreFile mockSf = mock(StoreFile.class);
    StoreFile.Reader reader = mock(StoreFile.Reader.class);
    String stringPath = "/hbase/testTable/regionA/"
        + RandomStringUtils.random(FILENAME_LENGTH, 0, 0, true, true, null, random);
    Path path = new Path(stringPath);




    when(reader.getSequenceID()).thenReturn(seqId);
    when(reader.getTotalUncompressedBytes()).thenReturn(sizeInBytes);
    when(reader.length()).thenReturn(sizeInBytes);


    when(mockSf.getPath()).thenReturn(path);
    when(mockSf.excludeFromMinorCompaction()).thenReturn(false);
    when(mockSf.isReference()).thenReturn(false); // TODO come back to
    // this when selection takes this into account
    when(mockSf.getReader()).thenReturn(reader);
    String toString = Objects.toStringHelper("MockStoreFile")
        .add("isReference", false)
        .add("fileSize", StringUtils.humanReadableInt(sizeInBytes))
        .add("seqId", seqId)
        .add("path", stringPath).toString();
    when(mockSf.toString()).thenReturn(toString);


    return mockSf;
  }

View Full Code Here

      ArrayList<StoreFile> candidateFiles, final List<StoreFile> filesCompacting) {
    // candidates = all storefiles not already in compaction queue
    if (!filesCompacting.isEmpty()) {
      // exclude all files older than the newest file we're currently
      // compacting. this allows us to preserve contiguity (HBASE-2856)
      StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
      int idx = candidateFiles.indexOf(last);
      Preconditions.checkArgument(idx != -1);
      candidateFiles.subList(0, idx + 1).clear();
    }
    return candidateFiles;

View Full Code Here

    // get store file sizes for incremental compacting selection.
    final int countOfFiles = candidates.size();
    long[] fileSizes = new long[countOfFiles];
    long[] sumSize = new long[countOfFiles];
    for (int i = countOfFiles - 1; i >= 0; --i) {
      StoreFile file = candidates.get(i);
      fileSizes[i] = file.getReader().length();
      // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
      int tooFar = i + comConf.getMaxFilesToCompact() - 1;
      sumSize[i] = fileSizes[i]
        + ((i + 1 < countOfFiles) ? sumSize[i + 1] : 0)
        - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);

View Full Code Here

    if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) {
      // Major compaction time has elapsed.
      long cfTtl = this.storeConfigInfo.getStoreFileTtl();
      if (filesToCompact.size() == 1) {
        // Single file
        StoreFile sf = filesToCompact.iterator().next();
        Long minTimestamp = sf.getMinimumTimestamp();
        long oldest = (minTimestamp == null)
            ? Long.MIN_VALUE
            : now - minTimestamp.longValue();
        if (sf.isMajorCompaction() &&
            (cfTtl == HConstants.FOREVER || oldest < cfTtl)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("Skipping major compaction of " + this +
                " because one (major) compacted file only and oldestTime " +
                oldest + "ms is < ttl=" + cfTtl);

View Full Code Here

    return createMockStoreFile(sizeMb, -1l);
  }




  private StoreFile createMockStoreFile(int sizeMb, long seqId) {
    StoreFile mockSf = mock(StoreFile.class);
    StoreFile.Reader reader = mock(StoreFile.Reader.class);
    String stringPath = "/hbase/" + RandomStringUtils.random(10, 0, 0, true, true, null, random);
    Path path = new Path(stringPath);


    when(reader.getSequenceID()).thenReturn(seqId);
    when(reader.getTotalUncompressedBytes()).thenReturn(Long.valueOf(sizeMb));
    when(reader.length()).thenReturn(Long.valueOf(sizeMb));


    when(mockSf.getPath()).thenReturn(path);
    when(mockSf.excludeFromMinorCompaction()).thenReturn(false);
    when(mockSf.isReference()).thenReturn(false); // TODO come back to
                                                  // this when selection takes this into account
    when(mockSf.getReader()).thenReturn(reader);
    String toString = Objects.toStringHelper("MockStoreFile")
                              .add("isReference", false)
                              .add("fileSize", sizeMb)
                              .add("seqId", seqId)
                              .add("path", stringPath).toString();
    when(mockSf.toString()).thenReturn(toString);


    return mockSf;
  }

View Full Code Here


  @Test
  public void testWithReferences() throws Exception {
    StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create());
    StripeCompactor sc = mock(StripeCompactor.class);
    StoreFile ref = createFile();
    when(ref.isReference()).thenReturn(true);
    StripeInformationProvider si = mock(StripeInformationProvider.class);
    Collection<StoreFile> sfs = al(ref, createFile());
    when(si.getStorefiles()).thenReturn(sfs);


    assertTrue(policy.needsCompactions(si, al()));

View Full Code Here

    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    long now = defaultTtl + 2;
    edge.setValue(now);
    EnvironmentEdgeManager.injectEdge(edge);
    try {
      StoreFile expiredFile = createFile(), notExpiredFile = createFile();
      when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
      when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
      List<StoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
      List<StoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);
      List<StoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile);

View Full Code Here

    ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
    long now = defaultTtl + 2;
    edge.setValue(now);
    EnvironmentEdgeManager.injectEdge(edge);
    try {
      StoreFile expiredFile = createFile(), notExpiredFile = createFile();
      when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
      when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
      List<StoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
      List<StoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);


      StripeCompactionPolicy policy =

View Full Code Here

    when(si.getLevel0Files()).thenReturn(l0Files);
    return si;
  }


  private static StoreFile createFile(long size) throws Exception {
    StoreFile sf = mock(StoreFile.class);
    when(sf.getPath()).thenReturn(new Path("moo"));
    StoreFile.Reader r = mock(StoreFile.Reader.class);
    when(r.getEntries()).thenReturn(size);
    when(r.length()).thenReturn(size);
    when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
    when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
    when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(
      mock(StoreFileScanner.class));
    when(sf.getReader()).thenReturn(r);
    when(sf.createReader()).thenReturn(r);
    return sf;
  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.hbase.regionserver.StoreFile$WriterBuilder

com.datastax.spark.connector.writer.WriteConf

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.hbase.io.HalfHFileReader

org.apache.hadoop.hbase.io.HalfStoreFileReader

org.apache.hadoop.hbase.io.hfile.HFile.Reader

org.apache.hadoop.hbase.io.hfile.LruBlockCache

org.apache.hadoop.hbase.io.HFileLink

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.