*

* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This class is responsible for splitting up a bunch of regionserver commit log

* files that are no longer being written to, into new files, one per region for

* region to replay on startup. Delete the old log files when finished.

*/

  static final Log LOG = LogFactory.getLog(HLogSplitter.class);

  // Parameters for split process

  protected final Path rootDir;

  protected final FileSystem fs;

  protected final Configuration conf;

  // Major subcomponents of the split process.

  // These are separated into inner classes to make testing easier.

  OutputSink outputSink;

  EntryBuffers entryBuffers;

  private Set<TableName> disablingOrDisabledTables =

      new HashSet<TableName>();

  private BaseCoordinatedStateManager csm;

  // If an exception is thrown by one of the other threads, it will be

  // stored here.

  protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();

  // Wait/notify for when data has been produced by the reader thread,

  // consumed by the reader thread, or an exception occurred

  final Object dataAvailable = new Object();

  private MonitoredTask status;

  // For checking the latest flushed sequence id

  protected final LastSequenceId sequenceIdChecker;

  protected boolean distributedLogReplay;

  // Map encodedRegionName -> lastFlushedSequenceId

  protected Map<String, Long> lastFlushedSequenceIds = new ConcurrentHashMap<String, Long>();

  // Map encodedRegionName -> maxSeqIdInStores

  protected Map<String, Map<byte[], Long>> regionMaxSeqIdInStores =

      new ConcurrentHashMap<String, Map<byte[], Long>>();

  // Failed region server that the wal file being split belongs to

  protected String failedServerName = "";

  // Number of writer threads

  private final int numWriterThreads;

  // Min batch size when replay WAL edits

  private final int minBatchSize;

  HLogSplitter(Configuration conf, Path rootDir,

      FileSystem fs, LastSequenceId idChecker,

      CoordinatedStateManager csm, RecoveryMode mode) {

    this.conf = HBaseConfiguration.create(conf);

    String codecClassName = conf

        .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());

    this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);

    this.rootDir = rootDir;

    this.fs = fs;

    this.sequenceIdChecker = idChecker;

    this.csm = (BaseCoordinatedStateManager)csm;

    entryBuffers = new EntryBuffers(

        this.conf.getInt("hbase.regionserver.hlog.splitlog.buffersize",

            128*1024*1024));

    // a larger minBatchSize may slow down recovery because replay writer has to wait for

    // enough edits before replaying them

    this.minBatchSize = this.conf.getInt("hbase.regionserver.wal.logreplay.batch.size", 64);

    this.distributedLogReplay = (RecoveryMode.LOG_REPLAY == mode);

    this.numWriterThreads = this.conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);

    if (csm != null && this.distributedLogReplay) {

      outputSink = new LogReplayOutputSink(numWriterThreads);

    } else {

      if (this.distributedLogReplay) {

        LOG.info("ZooKeeperWatcher is passed in as NULL so disable distrubitedLogRepaly.");

      }

      this.distributedLogReplay = false;

      outputSink = new LogRecoveredEditsOutputSink(numWriterThreads);

    }

  }

  /**

   * Splits a HLog file into region's recovered-edits directory.

   * This is the main entry point for distributed log splitting from SplitLogWorker.

   * <p>

   * If the log file has N regions then N recovered.edits files will be produced.

   * <p>

   * @param rootDir

   * @param logfile

   * @param fs

   * @param conf

   * @param reporter

   * @param idChecker

   * @param cp coordination state manager

   * @return false if it is interrupted by the progress-able.

   * @throws IOException

   */

  public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,

      Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,

      CoordinatedStateManager cp, RecoveryMode mode) throws IOException {

    HLogSplitter s = new HLogSplitter(conf, rootDir, fs, idChecker, cp, mode);

    return s.splitLogFile(logfile, reporter);

  }

  // A wrapper to split one log folder using the method used by distributed

  // log splitting. Used by tools and unit tests. It should be package private.

  // It is public only because TestWALObserver is in a different package,

  // which uses this method to to log splitting.

  public static List<Path> split(Path rootDir, Path logDir, Path oldLogDir,

      FileSystem fs, Configuration conf) throws IOException {

    FileStatus[] logfiles = fs.listStatus(logDir);

    List<Path> splits = new ArrayList<Path>();

    if (logfiles != null && logfiles.length > 0) {

      for (FileStatus logfile: logfiles) {

        HLogSplitter s =

            new HLogSplitter(conf, rootDir, fs, null, null, RecoveryMode.LOG_SPLITTING);

        if (s.splitLogFile(logfile, null)) {

          finishSplitLogFile(rootDir, oldLogDir, logfile.getPath(), conf);

          if (s.outputSink.splits != null) {

            splits.addAll(s.outputSink.splits);

          }

        }

      }

    }

    if (!fs.delete(logDir, true)) {

      throw new IOException("Unable to delete src dir: " + logDir);

    }

    return splits;

  }

  // The real log splitter. It just splits one log file.

  boolean splitLogFile(FileStatus logfile,

      CancelableProgressable reporter) throws IOException {

    boolean isCorrupted = false;

    Preconditions.checkState(status == null);

    boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors",

      HLog.SPLIT_SKIP_ERRORS_DEFAULT);

    int interval = conf.getInt("hbase.splitlog.report.interval.loglines", 1024);

    Path logPath = logfile.getPath();

    boolean outputSinkStarted = false;

    boolean progress_failed = false;

    int editsCount = 0;

    int editsSkipped = 0;

    status =

        TaskMonitor.get().createStatus(

          "Splitting log file " + logfile.getPath() + "into a temporary staging area.");

    try {

      long logLength = logfile.getLen();

      LOG.info("Splitting hlog: " + logPath + ", length=" + logLength);

      LOG.info("DistributedLogReplay = " + this.distributedLogReplay);

      status.setStatus("Opening log file");

      if (reporter != null && !reporter.progress()) {

        progress_failed = true;

        return false;

      }

      Reader in = null;

      try {

        in = getReader(fs, logfile, conf, skipErrors, reporter);

      } catch (CorruptedLogFileException e) {

        LOG.warn("Could not get reader, corrupted log file " + logPath, e);

        ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);

        isCorrupted = true;

      }

      if (in == null) {

        LOG.warn("Nothing to split in log file " + logPath);

        return true;

      }

      if(csm != null) {

        try {

          TableStateManager tsm = csm.getTableStateManager();

          disablingOrDisabledTables = tsm.getTablesInStates(

            ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING);

        } catch (CoordinatedStateException e) {

          throw new IOException("Can't get disabling/disabled tables", e);

        }

      }

      int numOpenedFilesBeforeReporting = conf.getInt("hbase.splitlog.report.openedfiles", 3);

      int numOpenedFilesLastCheck = 0;

      outputSink.setReporter(reporter);

      outputSink.startWriterThreads();

      outputSinkStarted = true;

      Entry entry;

      Long lastFlushedSequenceId = -1L;

      ServerName serverName = HLogUtil.getServerNameFromHLogDirectoryName(logPath);

      failedServerName = (serverName == null) ? "" : serverName.getServerName();

      while ((entry = getNextLogLine(in, logPath, skipErrors)) != null) {

        byte[] region = entry.getKey().getEncodedRegionName();

        String key = Bytes.toString(region);

        lastFlushedSequenceId = lastFlushedSequenceIds.get(key);

        if (lastFlushedSequenceId == null) {

          if (this.distributedLogReplay) {

            RegionStoreSequenceIds ids =

                csm.getSplitLogWorkerCoordination().getRegionFlushedSequenceId(failedServerName,

                  key);

            if (ids != null) {

              lastFlushedSequenceId = ids.getLastFlushedSequenceId();

            }

          } else if (sequenceIdChecker != null) {

            lastFlushedSequenceId = sequenceIdChecker.getLastSequenceId(region);

          }

          if (lastFlushedSequenceId == null) {

            lastFlushedSequenceId = -1L;

          }

          lastFlushedSequenceIds.put(key, lastFlushedSequenceId);

        }

        if (lastFlushedSequenceId >= entry.getKey().getLogSeqNum()) {

          editsSkipped++;

          continue;

        }

        entryBuffers.appendEntry(entry);

        editsCount++;

        int moreWritersFromLastCheck = this.getNumOpenWriters() - numOpenedFilesLastCheck;

        // If sufficient edits have passed, check if we should report progress.

        if (editsCount % interval == 0

            || moreWritersFromLastCheck > numOpenedFilesBeforeReporting) {

          numOpenedFilesLastCheck = this.getNumOpenWriters();

          String countsStr = (editsCount - (editsSkipped + outputSink.getSkippedEdits()))

              + " edits, skipped " + editsSkipped + " edits.";

          status.setStatus("Split " + countsStr);

          if (reporter != null && !reporter.progress()) {

            progress_failed = true;

            return false;

          }

        }

      }

    } catch (InterruptedException ie) {

      IOException iie = new InterruptedIOException();

      iie.initCause(ie);

      throw iie;

    } catch (CorruptedLogFileException e) {

      LOG.warn("Could not parse, corrupted log file " + logPath, e);

      csm.getSplitLogWorkerCoordination().markCorrupted(rootDir,

        logfile.getPath().getName(), fs);

      isCorrupted = true;

    } catch (IOException e) {

      e = RemoteExceptionHandler.checkIOException(e);

      throw e;

    } finally {

      LOG.debug("Finishing writing output logs and closing down.");

      try {

        if (outputSinkStarted) {

          // Set progress_failed to true as the immediate following statement will reset its value

          // when finishWritingAndClose() throws exception, progress_failed has the right value

          progress_failed = true;

          progress_failed = outputSink.finishWritingAndClose() == null;

        }

      } finally {

        String msg =

            "Processed " + editsCount + " edits across " + outputSink.getNumberOfRecoveredRegions()

                + " regions; log file=" + logPath + " is corrupted = " + isCorrupted

                + " progress failed = " + progress_failed;

        LOG.info(msg);

        status.markComplete(msg);

      }

    }

    return !progress_failed;

  }

  /**

   * Completes the work done by splitLogFile by archiving logs

   * <p>

   * It is invoked by SplitLogManager once it knows that one of the

   * SplitLogWorkers have completed the splitLogFile() part. If the master

   * crashes then this function might get called multiple times.

   * <p>

   * @param logfile

   * @param conf

   * @throws IOException

   */

  public static void finishSplitLogFile(String logfile,

      Configuration conf)  throws IOException {

    Path rootdir = FSUtils.getRootDir(conf);

    Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME);

    Path logPath;

    if (FSUtils.isStartingWithPath(rootdir, logfile)) {

      logPath = new Path(logfile);

    } else {

      logPath = new Path(rootdir, logfile);

    }

    finishSplitLogFile(rootdir, oldLogDir, logPath, conf);

  }

  static void finishSplitLogFile(Path rootdir, Path oldLogDir,

      Path logPath, Configuration conf) throws IOException {

    List<Path> processedLogs = new ArrayList<Path>();

    List<Path> corruptedLogs = new ArrayList<Path>();

    FileSystem fs;

    fs = rootdir.getFileSystem(conf);

    if (ZKSplitLog.isCorrupted(rootdir, logPath.getName(), fs)) {

      corruptedLogs.add(logPath);

    } else {

      processedLogs.add(logPath);

    }

    archiveLogs(corruptedLogs, processedLogs, oldLogDir, fs, conf);

    Path stagingDir = ZKSplitLog.getSplitLogDir(rootdir, logPath.getName());

    fs.delete(stagingDir, true);

  }

  /**

   * Moves processed logs to a oldLogDir after successful processing Moves

   * corrupted logs (any log that couldn't be successfully parsed to corruptDir

   * (.corrupt) for later investigation

   *

   * @param corruptedLogs

   * @param processedLogs

   * @param oldLogDir

   * @param fs

   * @param conf

   * @throws IOException

   */

  private static void archiveLogs(

      final List<Path> corruptedLogs,

      final List<Path> processedLogs, final Path oldLogDir,

      final FileSystem fs, final Configuration conf) throws IOException {

    final Path corruptDir = new Path(FSUtils.getRootDir(conf), conf.get(

        "hbase.regionserver.hlog.splitlog.corrupt.dir",  HConstants.CORRUPT_DIR_NAME));

    if (!fs.mkdirs(corruptDir)) {

      LOG.info("Unable to mkdir " + corruptDir);

    }

    fs.mkdirs(oldLogDir);

    // this method can get restarted or called multiple times for archiving

    // the same log files.

    for (Path corrupted : corruptedLogs) {

      Path p = new Path(corruptDir, corrupted.getName());

      if (fs.exists(corrupted)) {

        if (!fs.rename(corrupted, p)) {

          LOG.warn("Unable to move corrupted log " + corrupted + " to " + p);

        } else {

          LOG.warn("Moved corrupted log " + corrupted + " to " + p);

        }

      }

    }

    for (Path p : processedLogs) {

      Path newPath = FSHLog.getHLogArchivePath(oldLogDir, p);

      if (fs.exists(p)) {

        if (!FSUtils.renameAndSetModifyTime(fs, p, newPath)) {

          LOG.warn("Unable to move  " + p + " to " + newPath);

        } else {

          LOG.info("Archived processed log " + p + " to " + newPath);

        }

      }

    }

  }

  /**

   * Path to a file under RECOVERED_EDITS_DIR directory of the region found in

   * <code>logEntry</code> named for the sequenceid in the passed

   * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.

   * This method also ensures existence of RECOVERED_EDITS_DIR under the region

   * creating it if necessary.

   * @param fs

   * @param logEntry

   * @param rootDir HBase root dir.

   * @return Path to file into which to dump split log edits.

   * @throws IOException

   */

  @SuppressWarnings("deprecation")

  static Path getRegionSplitEditsPath(final FileSystem fs,

      final Entry logEntry, final Path rootDir, boolean isCreate)

  throws IOException {

    Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTablename());

    String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());

    Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);

    Path dir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);

    if (!fs.exists(regiondir)) {

      LOG.info("This region's directory doesn't exist: "

          + regiondir.toString() + ". It is very likely that it was" +

          " already split so it's safe to discard those edits.");

      return null;

    }

    if (fs.exists(dir) && fs.isFile(dir)) {

      Path tmp = new Path("/tmp");

      if (!fs.exists(tmp)) {

        fs.mkdirs(tmp);

      }

      tmp = new Path(tmp,

        HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);

      LOG.warn("Found existing old file: " + dir + ". It could be some "

        + "leftover of an old installation. It should be a folder instead. "

        + "So moving it to " + tmp);

      if (!fs.rename(dir, tmp)) {

        LOG.warn("Failed to sideline old file " + dir);

      }

    }

    if (isCreate && !fs.exists(dir)) {

      if (!fs.mkdirs(dir)) LOG.warn("mkdir failed on " + dir);

    }

    // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure

    // region's replayRecoveredEdits will not delete it

    String fileName = formatRecoveredEditsFileName(logEntry.getKey().getLogSeqNum());

    fileName = getTmpRecoveredEditsFileName(fileName);

    return new Path(dir, fileName);

  }

  static String getTmpRecoveredEditsFileName(String fileName) {

    return fileName + HLog.RECOVERED_LOG_TMPFILE_SUFFIX;

  }

  /**

   * Get the completed recovered edits file path, renaming it to be by last edit

   * in the file from its first edit. Then we could use the name to skip

   * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.

   * @param srcPath

   * @param maximumEditLogSeqNum

   * @return dstPath take file's last edit log seq num as the name

   */

  static Path getCompletedRecoveredEditsFilePath(Path srcPath,

      Long maximumEditLogSeqNum) {

    String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);

    return new Path(srcPath.getParent(), fileName);

  }

  static String formatRecoveredEditsFileName(final long seqid) {

    return String.format("%019d", seqid);

  }

  /**

   * Create a new {@link Reader} for reading logs to split.

   *

   * @param fs

   * @param file

   * @param conf

   * @return A new Reader instance

   * @throws IOException

   * @throws CorruptedLogFileException

   */

  protected Reader getReader(FileSystem fs, FileStatus file, Configuration conf,

      boolean skipErrors, CancelableProgressable reporter)

      throws IOException, CorruptedLogFileException {

    Path path = file.getPath();

    long length = file.getLen();

    Reader in;

    // Check for possibly empty file. With appends, currently Hadoop reports a

    // zero length even if the file has been sync'd. Revisit if HDFS-376 or

    // HDFS-878 is committed.

    if (length <= 0) {

      LOG.warn("File " + path + " might be still open, length is 0");

    }

    try {

      FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);

      try {

        in = getReader(fs, path, conf, reporter);

      } catch (EOFException e) {

        if (length <= 0) {

          // TODO should we ignore an empty, not-last log file if skip.errors

          // is false? Either way, the caller should decide what to do. E.g.

          // ignore if this is the last log in sequence.

          // TODO is this scenario still possible if the log has been

          // recovered (i.e. closed)

          LOG.warn("Could not open " + path + " for reading. File is empty", e);

          return null;

        } else {

          // EOFException being ignored

          return null;

        }

      }

    } catch (IOException e) {

      if (e instanceof FileNotFoundException) {

        // A wal file may not exist anymore. Nothing can be recovered so move on

        LOG.warn("File " + path + " doesn't exist anymore.", e);

        return null;

      }

      if (!skipErrors || e instanceof InterruptedIOException) {

        throw e; // Don't mark the file corrupted if interrupted, or not skipErrors

      }

      CorruptedLogFileException t =

        new CorruptedLogFileException("skipErrors=true Could not open hlog " +

            path + " ignoring");

      t.initCause(e);

      throw t;

    }

    return in;

  }

  static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)

  throws CorruptedLogFileException, IOException {

    try {

      return in.next();

    } catch (EOFException eof) {

      // truncated files are expected if a RS crashes (see HBASE-2643)

      LOG.info("EOF from hlog " + path + ".  continuing");

      return null;

    } catch (IOException e) {

      // If the IOE resulted from bad file format,

      // then this problem is idempotent and retrying won't help

      if (e.getCause() != null &&

          (e.getCause() instanceof ParseException ||

           e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {

        LOG.warn("Parse exception " + e.getCause().toString() + " from hlog "

           + path + ".  continuing");

        return null;

      }

      if (!skipErrors) {

        throw e;

      }

      CorruptedLogFileException t =

        new CorruptedLogFileException("skipErrors=true Ignoring exception" +

            " while parsing hlog " + path + ". Marking as corrupted");

      t.initCause(e);

      throw t;

    }

  }

  private void writerThreadError(Throwable t) {

    thrown.compareAndSet(null, t);

  }

  /**

   * Check for errors in the writer threads. If any is found, rethrow it.

   */

  private void checkForErrors() throws IOException {

    Throwable thrown = this.thrown.get();

    if (thrown == null) return;

    if (thrown instanceof IOException) {

      throw new IOException(thrown);

    } else {

      throw new RuntimeException(thrown);

    }

  }

  /**

   * Create a new {@link Writer} for writing log splits.

   */

  protected Writer createWriter(FileSystem fs, Path logfile, Configuration conf)

      throws IOException {

    return HLogFactory.createRecoveredEditsWriter(fs, logfile, conf);

  }

  /**

   * Create a new {@link Reader} for reading logs to split.

   */

  protected Reader getReader(FileSystem fs, Path curLogFile,

      Configuration conf, CancelableProgressable reporter) throws IOException {

    return HLogFactory.createReader(fs, curLogFile, conf, reporter);

  }

  /**

   * Get current open writers

   */

  private int getNumOpenWriters() {

    int result = 0;

    if (this.outputSink != null) {

      result += this.outputSink.getNumOpenWriters();

    }

    return result;

  }

  /**

   * Class which accumulates edits and separates them into a buffer per region

   * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses

   * a predefined threshold.

   *

   * Writer threads then pull region-specific buffers from this class.

   */

  class EntryBuffers {

    Map<byte[], RegionEntryBuffer> buffers =

      new TreeMap<byte[], RegionEntryBuffer>(Bytes.BYTES_COMPARATOR);

    /* Track which regions are currently in the middle of writing. We don't allow

       an IO thread to pick up bytes from a region if we're already writing

       data for that region in a different IO thread. */

    Set<byte[]> currentlyWriting = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);

    long totalBuffered = 0;

    long maxHeapUsage;

    EntryBuffers(long maxHeapUsage) {

      this.maxHeapUsage = maxHeapUsage;

    }

    /**

     * Append a log entry into the corresponding region buffer.

     * Blocks if the total heap usage has crossed the specified threshold.

     *

     * @throws InterruptedException

     * @throws IOException

     */

    void appendEntry(Entry entry) throws InterruptedException, IOException {

      HLogKey key = entry.getKey();

      RegionEntryBuffer buffer;

      long incrHeap;

      synchronized (this) {

        buffer = buffers.get(key.getEncodedRegionName());

        if (buffer == null) {

          buffer = new RegionEntryBuffer(key.getTablename(), key.getEncodedRegionName());

          buffers.put(key.getEncodedRegionName(), buffer);

        }

        incrHeap= buffer.appendEntry(entry);

      }

      // If we crossed the chunk threshold, wait for more space to be available

      synchronized (dataAvailable) {

        totalBuffered += incrHeap;

        while (totalBuffered > maxHeapUsage && thrown.get() == null) {

          LOG.debug("Used " + totalBuffered + " bytes of buffered edits, waiting for IO threads...");

          dataAvailable.wait(2000);

        }

        dataAvailable.notifyAll();

      }

      checkForErrors();

    }

    /**

     * @return RegionEntryBuffer a buffer of edits to be written or replayed.

     */

    synchronized RegionEntryBuffer getChunkToWrite() {

      long biggestSize = 0;

      byte[] biggestBufferKey = null;

      for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {

        long size = entry.getValue().heapSize();

        if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {

          biggestSize = size;

          biggestBufferKey = entry.getKey();

        }

      }

      if (biggestBufferKey == null) {

        return null;

      }

      RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);

      currentlyWriting.add(biggestBufferKey);

      return buffer;

    }

    void doneWriting(RegionEntryBuffer buffer) {

      synchronized (this) {

        boolean removed = currentlyWriting.remove(buffer.encodedRegionName);

        assert removed;

      }

      long size = buffer.heapSize();

      synchronized (dataAvailable) {

        totalBuffered -= size;

        // We may unblock writers

        dataAvailable.notifyAll();

      }

    }

    synchronized boolean isRegionCurrentlyWriting(byte[] region) {

      return currentlyWriting.contains(region);

    }

  }

  /**

   * A buffer of some number of edits for a given region.

   * This accumulates edits and also provides a memory optimization in order to

   * share a single byte array instance for the table and region name.

   * Also tracks memory usage of the accumulated edits.

   */

  static class RegionEntryBuffer implements HeapSize {

    long heapInBuffer = 0;

    List<Entry> entryBuffer;

    TableName tableName;

    byte[] encodedRegionName;

    RegionEntryBuffer(TableName tableName, byte[] region) {

      this.tableName = tableName;

      this.encodedRegionName = region;

      this.entryBuffer = new LinkedList<Entry>();

    }

    long appendEntry(Entry entry) {

      internify(entry);

      entryBuffer.add(entry);

      long incrHeap = entry.getEdit().heapSize() +

        ClassSize.align(2 * ClassSize.REFERENCE) + // HLogKey pointers

        0; // TODO linkedlist entry

      heapInBuffer += incrHeap;

      return incrHeap;

    }

    private void internify(Entry entry) {

      HLogKey k = entry.getKey();

      k.internTableName(this.tableName);

      k.internEncodedRegionName(this.encodedRegionName);

    }

    @Override

    public long heapSize() {

      return heapInBuffer;

    }

  }

  class WriterThread extends Thread {

    private volatile boolean shouldStop = false;

    private OutputSink outputSink = null;

    WriterThread(OutputSink sink, int i) {

      super(Thread.currentThread().getName() + "-Writer-" + i);

      outputSink = sink;

    }

    @Override

    public void run()  {

      try {

        doRun();

      } catch (Throwable t) {

        LOG.error("Exiting thread", t);

        writerThreadError(t);

      }

    }

    private void doRun() throws IOException {

      LOG.debug("Writer thread " + this + ": starting");

      while (true) {

        RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();

        if (buffer == null) {

          // No data currently available, wait on some more to show up

          synchronized (dataAvailable) {

            if (shouldStop && !this.outputSink.flush()) {

              return;

            }

            try {

              dataAvailable.wait(500);

            } catch (InterruptedException ie) {

              if (!shouldStop) {

                throw new RuntimeException(ie);

              }

            }

          }

          continue;

        }

        assert buffer != null;

        try {

          writeBuffer(buffer);

        } finally {

          entryBuffers.doneWriting(buffer);

        }

      }

    }

    private void writeBuffer(RegionEntryBuffer buffer) throws IOException {

      outputSink.append(buffer);

    }

    void finish() {

      synchronized (dataAvailable) {

        shouldStop = true;

        dataAvailable.notifyAll();

      }

    }

  }

  /**

   * The following class is an abstraction class to provide a common interface to support both

   * existing recovered edits file sink and region server WAL edits replay sink

   */

   abstract class OutputSink {

    protected Map<byte[], SinkWriter> writers = Collections

        .synchronizedMap(new TreeMap<byte[], SinkWriter>(Bytes.BYTES_COMPARATOR));;

    protected final Map<byte[], Long> regionMaximumEditLogSeqNum = Collections

        .synchronizedMap(new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR));

    protected final List<WriterThread> writerThreads = Lists.newArrayList();

    /* Set of regions which we've decided should not output edits */

    protected final Set<byte[]> blacklistedRegions = Collections

        .synchronizedSet(new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR));

    protected boolean closeAndCleanCompleted = false;

    protected boolean writersClosed = false;

    protected final int numThreads;

    protected CancelableProgressable reporter = null;

    protected AtomicLong skippedEdits = new AtomicLong();

    protected List<Path> splits = null;

    public OutputSink(int numWriters) {

      numThreads = numWriters;

    }

    void setReporter(CancelableProgressable reporter) {

      this.reporter = reporter;

    }

    /**

     * Start the threads that will pump data from the entryBuffers to the output files.

     */

    synchronized void startWriterThreads() {

      for (int i = 0; i < numThreads; i++) {

        WriterThread t = new WriterThread(this, i);

        t.start();

        writerThreads.add(t);

      }

    }

    /**

     *

     * Update region's maximum edit log SeqNum.

     */

    void updateRegionMaximumEditLogSeqNum(Entry entry) {

      synchronized (regionMaximumEditLogSeqNum) {

        Long currentMaxSeqNum = regionMaximumEditLogSeqNum.get(entry.getKey()

            .getEncodedRegionName());

        if (currentMaxSeqNum == null || entry.getKey().getLogSeqNum() > currentMaxSeqNum) {

          regionMaximumEditLogSeqNum.put(entry.getKey().getEncodedRegionName(), entry.getKey()

              .getLogSeqNum());

        }

      }

    }

    Long getRegionMaximumEditLogSeqNum(byte[] region) {

      return regionMaximumEditLogSeqNum.get(region);

    }

    /**

     * @return the number of currently opened writers

     */

    int getNumOpenWriters() {

      return this.writers.size();

    }

    long getSkippedEdits() {

      return this.skippedEdits.get();

    }

    /**

     * Wait for writer threads to dump all info to the sink

     * @return true when there is no error

     * @throws IOException

     */

    protected boolean finishWriting() throws IOException {

      LOG.debug("Waiting for split writer threads to finish");

      boolean progress_failed = false;

      for (WriterThread t : writerThreads) {

        t.finish();

      }

      for (WriterThread t : writerThreads) {

        if (!progress_failed && reporter != null && !reporter.progress()) {

          progress_failed = true;

        }

        try {

          t.join();

        } catch (InterruptedException ie) {

          IOException iie = new InterruptedIOException();

          iie.initCause(ie);

          throw iie;

        }

      }

      checkForErrors();

      LOG.info("Split writers finished");

      return (!progress_failed);

    }

    abstract List<Path> finishWritingAndClose() throws IOException;

    /**

     * @return a map from encoded region ID to the number of edits written out for that region.

     */

    abstract Map<byte[], Long> getOutputCounts();

    /**

     * @return number of regions we've recovered

     */

    abstract int getNumberOfRecoveredRegions();

    /**

     * @param buffer A WAL Edit Entry

     * @throws IOException

     */

    abstract void append(RegionEntryBuffer buffer) throws IOException;

    /**

     * WriterThread call this function to help flush internal remaining edits in buffer before close

     * @return true when underlying sink has something to flush

     */

    protected boolean flush() throws IOException {

      return false;

    }

  }

  /**

   * Class that manages the output streams from the log splitting process.

   */

  class LogRecoveredEditsOutputSink extends OutputSink {

    public LogRecoveredEditsOutputSink(int numWriters) {

      // More threads could potentially write faster at the expense

      // of causing more disk seeks as the logs are split.

      // 3. After a certain setting (probably around 3) the

      // process will be bound on the reader in the current

      // implementation anyway.

      super(numWriters);

    }

    /**

     * @return null if failed to report progress

     * @throws IOException

     */

    @Override

    List<Path> finishWritingAndClose() throws IOException {

      boolean isSuccessful = false;

      List<Path> result = null;

      try {

        isSuccessful = finishWriting();

      } finally {

        result = close();

        List<IOException> thrown = closeLogWriters(null);

        if (thrown != null && !thrown.isEmpty()) {

          throw MultipleIOException.createIOException(thrown);

        }

      }

      if (isSuccessful) {

        splits = result;

      }

      return splits;

    }

    /**

     * Close all of the output streams.

     * @return the list of paths written.

     */

    private List<Path> close() throws IOException {

      Preconditions.checkState(!closeAndCleanCompleted);

      final List<Path> paths = new ArrayList<Path>();

      final List<IOException> thrown = Lists.newArrayList();

      ThreadPoolExecutor closeThreadPool = Threads.getBoundedCachedThreadPool(numThreads, 30L,

        TimeUnit.SECONDS, new ThreadFactory() {

          private int count = 1;

          @Override

          public Thread newThread(Runnable r) {

            Thread t = new Thread(r, "split-log-closeStream-" + count++);

            return t;

          }

        });