Package net.opentsdb.tools

Source Code of net.opentsdb.tools.UidManager

// This file is part of OpenTSDB.
// Copyright (C) 2010-2012  The OpenTSDB Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or (at your
// option) any later version.  This program is distributed in the hope that it
// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
// General Public License for more details.  You should have received a copy
// of the GNU Lesser General Public License along with this program.  If not,
// see <http://www.gnu.org/licenses/>.
package net.opentsdb.tools;

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.hbase.async.Bytes;
import org.hbase.async.GetRequest;
import org.hbase.async.HBaseClient;
import org.hbase.async.HBaseException;
import org.hbase.async.KeyValue;
import org.hbase.async.Scanner;

import net.opentsdb.core.TSDB;
import net.opentsdb.meta.TSMeta;
import net.opentsdb.uid.NoSuchUniqueId;
import net.opentsdb.uid.NoSuchUniqueName;
import net.opentsdb.uid.UniqueId;
import net.opentsdb.utils.Config;

/**
* Command line tool to manipulate UIDs.
* Can be used to find or assign UIDs.
*/
final class UidManager {

  private static final Logger LOG = LoggerFactory.getLogger(UidManager.class);

  /** Function used to convert a String to a byte[]. */
  private static final Method toBytes;
  /** Function used to convert a byte[] to a String. */
  private static final Method fromBytes;
  /** Charset used to convert Strings to byte arrays and back. */
  private static final Charset CHARSET;
  /** The single column family used by this class. */
  private static final byte[] ID_FAMILY;
  /** The single column family used by this class. */
  private static final byte[] NAME_FAMILY;
  /** Row key of the special row used to track the max ID already assigned. */
  private static final byte[] MAXID_ROW;
  static {
    final Class<UniqueId> uidclass = UniqueId.class;
    try {
      // Those are all implementation details so they're not part of the
      // interface.  We access them anyway using reflection.  I think this
      // is better than marking those public and adding a javadoc comment
      // "THIS IS INTERNAL DO NOT USE".  If only Java had C++'s "friend" or
      // a less stupid notion of a package.
      Field f;
      f = uidclass.getDeclaredField("CHARSET");
      f.setAccessible(true);
      CHARSET = (Charset) f.get(null);
      f = uidclass.getDeclaredField("ID_FAMILY");
      f.setAccessible(true);
      ID_FAMILY = (byte[]) f.get(null);
      f = uidclass.getDeclaredField("NAME_FAMILY");
      f.setAccessible(true);
      NAME_FAMILY = (byte[]) f.get(null);
      f = uidclass.getDeclaredField("MAXID_ROW");
      f.setAccessible(true);
      MAXID_ROW = (byte[]) f.get(null);
      toBytes = uidclass.getDeclaredMethod("toBytes", String.class);
      toBytes.setAccessible(true);
      fromBytes = uidclass.getDeclaredMethod("fromBytes", byte[].class);
      fromBytes.setAccessible(true);
    } catch (Exception e) {
      throw new RuntimeException("static initializer failed", e);
    }
  }

  /** Prints usage. */
  static void usage(final String errmsg) {
    usage(null, errmsg);
  }

  /** Prints usage. */
  static void usage(final ArgP argp, final String errmsg) {
    System.err.println(errmsg);
    System.err.println("Usage: uid <subcommand> args\n"
        + "Sub commands:\n"
        + "  grep [kind] <RE>: Finds matching IDs.\n"
        + "  assign <kind> <name> [names]:"
        + " Assign an ID for the given name(s).\n"
        + "  rename <kind> <name> <newname>: Renames this UID.\n"
        + "  fsck: Checks the consistency of UIDs.\n"
        + "  [kind] <name>: Lookup the ID of this name.\n"
        + "  [kind] <ID>: Lookup the name of this ID.\n"
        + "  metasync: Generates missing TSUID and UID meta entries, updates\n"
        + "            created timestamps\n"
        + "  metapurge: Removes meta data entries from the UID table\n"
        + "  treesync: Process all timeseries meta objects through tree rules\n"
        + "  treepurge <id> [definition]: Purge a tree and/or the branches\n"
        + "            from storage. Provide an integer Tree ID and optionally\n"
        + "            add \"true\" to delete the tree definition\n\n"
        + "Example values for [kind]:"
        + " metrics, tagk (tag name), tagv (tag value).");
    if (argp != null) {
      System.err.print(argp.usage());
    }
  }

  public static void main(String[] args) throws Exception {
    ArgP argp = new ArgP();
    CliOptions.addCommon(argp);
    CliOptions.addVerbose(argp);
    argp.addOption("--idwidth", "N",
                   "Number of bytes on which the UniqueId is encoded.");
    argp.addOption("--ignore-case",
                   "Ignore case distinctions when matching a regexp.");
    argp.addOption("-i", "Short for --ignore-case.");
    args = CliOptions.parse(argp, args);
    if (args == null) {
      usage(argp, "Invalid usage");
      System.exit(2);
    } else if (args.length < 1) {
      usage(argp, "Not enough arguments");
      System.exit(2);
    }  
    final short idwidth = (argp.has("--idwidth")
                           ? Short.parseShort(argp.get("--idwidth"))
                           : 3);
    if (idwidth <= 0) {
      usage(argp, "Negative or 0 --idwidth");
      System.exit(3);
    }
    final boolean ignorecase = argp.has("--ignore-case") || argp.has("-i");
   
    // get a config object
    Config config = CliOptions.getConfig(argp);
    final byte[] table = config.getString("tsd.storage.hbase.uid_table")
      .getBytes();
   
    final TSDB tsdb = new TSDB(config);
    tsdb.getClient().ensureTableExists(
        config.getString("tsd.storage.hbase.uid_table")).joinUninterruptibly();
    argp = null;
    int rc;
    try {
      rc = runCommand(tsdb, table, idwidth, ignorecase, args);
    } finally {
      try {
        tsdb.getClient().shutdown().joinUninterruptibly();
        LOG.info("Gracefully shutdown the TSD");
      } catch (Exception e) {
        LOG.error("Unexpected exception while shutting down", e);
        rc = 42;
      }
    }
    System.exit(rc);
  }

  private static int runCommand(final TSDB tsdb,
                                final byte[] table,
                                final short idwidth,
                                final boolean ignorecase,
                                final String[] args) {
    final int nargs = args.length;
    if (args[0].equals("grep")) {
      if (2 <= nargs && nargs <= 3) {
        try {
          return grep(tsdb.getClient(), table, ignorecase, args);
        } catch (HBaseException e) {
          return 3;
        }
      } else {
        usage("Wrong number of arguments");
        return 2;
      }
    } else if (args[0].equals("assign")) {
      if (nargs < 3) {
        usage("Wrong number of arguments");
        return 2;
      }
      return assign(tsdb.getClient(), table, idwidth, args);
    } else if (args[0].equals("rename")) {
      if (nargs != 4) {
        usage("Wrong number of arguments");
        return 2;
      }
      return rename(tsdb.getClient(), table, idwidth, args);
    } else if (args[0].equals("fsck")) {
      return fsck(tsdb.getClient(), table);
    } else if (args[0].equals("metasync")) {
      // check for the data table existence and initialize our plugins
      // so that update meta data can be pushed to search engines
      try {
        tsdb.getClient().ensureTableExists(
            tsdb.getConfig().getString(
                "tsd.storage.hbase.data_table")).joinUninterruptibly();
        tsdb.initializePlugins(false);
        return metaSync(tsdb);
      } catch (Exception e) {
        LOG.error("Unexpected exception", e);
        return 3;
      }     
    } else if (args[0].equals("metapurge")) {
      // check for the data table existence and initialize our plugins
      // so that update meta data can be pushed to search engines
      try {
        tsdb.getClient().ensureTableExists(
            tsdb.getConfig().getString(
                "tsd.storage.hbase.uid_table")).joinUninterruptibly();
        return metaPurge(tsdb);
      } catch (Exception e) {
        LOG.error("Unexpected exception", e);
        return 3;
      }     
    } else if (args[0].equals("treesync")) {
      // check for the UID table existence
      try {
        tsdb.getClient().ensureTableExists(
            tsdb.getConfig().getString(
                "tsd.storage.hbase.uid_table")).joinUninterruptibly();
        if (!tsdb.getConfig().enable_tree_processing()) {
          LOG.warn("Tree processing is disabled");
          return 0;
        }
        return treeSync(tsdb);
      } catch (Exception e) {
        LOG.error("Unexpected exception", e);
        return 3;
      }     
    } else if (args[0].equals("treepurge")) {
      if (nargs < 2) {
        usage("Wrong number of arguments");
        return 2;
      }
      try {
        tsdb.getClient().ensureTableExists(
            tsdb.getConfig().getString(
                "tsd.storage.hbase.uid_table")).joinUninterruptibly();
        final int tree_id = Integer.parseInt(args[1]);
        final boolean delete_definitions;
        if (nargs < 3) {
          delete_definitions = false;
        } else {
          final String delete_all = args[2];
          if (delete_all.toLowerCase().equals("true")) {
            delete_definitions = true;
          } else {
            delete_definitions = false;
          }
        }
        return purgeTree(tsdb, tree_id, delete_definitions);
      } catch (Exception e) {
        LOG.error("Unexpected exception", e);
        return 3;
      }     
    } else {
      if (1 <= nargs && nargs <= 2) {
        final String kind = nargs == 2 ? args[0] : null;
        try {
          final long id = Long.parseLong(args[nargs - 1]);
          return lookupId(tsdb.getClient(), table, idwidth, id, kind);
        } catch (NumberFormatException e) {
          return lookupName(tsdb.getClient(), table, idwidth,
              args[nargs - 1], kind);
        }
      } else {
        usage("Wrong number of arguments");
        return 2;
      }
    }
  }

  /**
   * Implements the {@code grep} subcommand.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param ignorecase Whether or not to ignore the case while grepping.
   * @param args Command line arguments ({@code [kind] RE}).
   * @return The exit status of the command (0 means at least 1 match).
   */
  private static int grep(final HBaseClient client,
                          final byte[] table,
                          final boolean ignorecase,
                          final String[] args) {
    final Scanner scanner = client.newScanner(table);
    scanner.setMaxNumRows(1024);
    String regexp;
    scanner.setFamily(ID_FAMILY);
    if (args.length == 3) {
      scanner.setQualifier(toBytes(args[1]));
      regexp = args[2];
    } else {
      regexp = args[1];
    }
    if (ignorecase) {
      regexp = "(?i)" + regexp;
    }
    scanner.setKeyRegexp(regexp, CHARSET);
    boolean found = false;
    try {
      ArrayList<ArrayList<KeyValue>> rows;
      while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
        for (final ArrayList<KeyValue> row : rows) {
          found |= printResult(row, ID_FAMILY, true);
        }
      }
    } catch (HBaseException e) {
      LOG.error("Error while scanning HBase, scanner=" + scanner, e);
      throw e;
    } catch (Exception e) {
      LOG.error("WTF?  Unexpected exception type, scanner=" + scanner, e);
      throw new AssertionError("Should never happen");
    }
    return found ? 0 : 1;
  }

  /**
   * Helper to print the cells in a given family for a given row, if any.
   * @param row The row to print.
   * @param family Only cells in this family (if any) will be printed.
   * @param formard If true, this row contains a forward mapping (name to ID).
   * Otherwise the row is assumed to contain a reverse mapping (ID to name).
   * @return {@code true} if at least one cell was printed.
   */
  private static boolean printResult(final ArrayList<KeyValue> row,
                                     final byte[] family,
                                     final boolean formard) {
    final byte[] key = row.get(0).key();
    String name = formard ? fromBytes(key) : null;
    String id = formard ? null : Arrays.toString(key);
    boolean printed = false;
    for (final KeyValue kv : row) {
      if (!Bytes.equals(kv.family(), family)) {
        continue;
      }
      printed = true;
      if (formard) {
        id = Arrays.toString(kv.value());
      } else {
        name = fromBytes(kv.value());
      }
      System.out.println(fromBytes(kv.qualifier()) + ' ' + name + ": " + id);
    }
    return printed;
  }

  /**
   * Implements the {@code assign} subcommand.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param args Command line arguments ({@code assign name [names]}).
   * @return The exit status of the command (0 means success).
   */
  private static int assign(final HBaseClient client,
                            final byte[] table,
                            final short idwidth,
                            final String[] args) {
    final UniqueId uid = new UniqueId(client, table, args[1], (int) idwidth);
    for (int i = 2; i < args.length; i++) {
      try {
        uid.getOrCreateId(args[i]);
        // Lookup again the ID we've just created and print it.
        extactLookupName(client, table, idwidth, args[1], args[i]);
      } catch (HBaseException e) {
        LOG.error("error while processing " + args[i], e);
        return 3;
      }
    }
    return 0;
  }

  /**
   * Implements the {@code rename} subcommand.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param args Command line arguments ({@code assign name [names]}).
   * @return The exit status of the command (0 means success).
   */
  private static int rename(final HBaseClient client,
                            final byte[] table,
                            final short idwidth,
                            final String[] args) {
    final String kind = args[1];
    final String oldname = args[2];
    final String newname = args[3];
    final UniqueId uid = new UniqueId(client, table, kind, (int) idwidth);
    try {
      uid.rename(oldname, newname);
    } catch (HBaseException e) {
      LOG.error("error while processing renaming " + oldname
                + " to " + newname, e);
      return 3;
    } catch (NoSuchUniqueName e) {
      LOG.error(e.getMessage());
      return 1;
    }
    System.out.println(kind + ' ' + oldname + " -> " + newname);
    return 0;
  }

  /**
   * Implements the {@code fsck} subcommand.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @return The exit status of the command (0 means success).
   */
  private static int fsck(final HBaseClient client, final byte[] table) {

    final class Uids {
      int errors;
      long maxid;
      short width;
      final HashMap<String, String> id2name = new HashMap<String, String>();
      final HashMap<String, String> name2id = new HashMap<String, String>();

      void error(final KeyValue kv, final String msg) {
        error(msg + ".  kv=" + kv);
      }

      void error(final String msg) {
        LOG.error(msg);
        errors++;
      }
    }

    final byte[] METRICS_META = "metric_meta".getBytes(CHARSET);
    final byte[] TAGK_META = "tagk_meta".getBytes(CHARSET);
    final byte[] TAGV_META = "tagv_meta".getBytes(CHARSET);
    final long start_time = System.nanoTime();
    final HashMap<String, Uids> name2uids = new HashMap<String, Uids>();
    final Scanner scanner = client.newScanner(table);
    scanner.setMaxNumRows(1024);
    int kvcount = 0;
    try {
      ArrayList<ArrayList<KeyValue>> rows;
      while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
        for (final ArrayList<KeyValue> row : rows) {
          for (final KeyValue kv : row) {
            kvcount++;
           
            // TODO - validate meta data in the future, for now skip it
            if (Bytes.equals(kv.qualifier(), TSMeta.META_QUALIFIER()) ||
                Bytes.equals(kv.qualifier(), TSMeta.COUNTER_QUALIFIER()) ||
                Bytes.equals(kv.qualifier(), METRICS_META) ||
                Bytes.equals(kv.qualifier(), TAGK_META) ||
                Bytes.equals(kv.qualifier(), TAGV_META)) {
              continue;
            }
           
            final String kind = fromBytes(kv.qualifier());
            Uids uids = name2uids.get(kind);
            if (uids == null) {
              uids = new Uids();
              name2uids.put(kind, uids);
            }
            final byte[] key = kv.key();
            final byte[] family = kv.family();
            final byte[] value = kv.value();
            if (Bytes.equals(key, MAXID_ROW)) {
              if (value.length != 8) {
                uids.error(kv, "Invalid maximum ID for " + kind
                           + ": should be on 8 bytes: ");
              } else {
                uids.maxid = Bytes.getLong(value);
                LOG.info("Maximum ID for " + kind + ": " + uids.maxid);
              }
            } else {
              short idwidth = 0;
              if (Bytes.equals(family, ID_FAMILY)) {
                idwidth = (short) value.length;
                final String skey = fromBytes(key);
                final String svalue = Arrays.toString(value);
                final String id = uids.name2id.put(skey, svalue);
                if (id != null) {
                  uids.error(kv, "Duplicate forward " + kind + " mapping: "
                             + skey + " -> " + id
                             + " and " + skey + " -> " + svalue);
                }
              } else if (Bytes.equals(family, NAME_FAMILY)) {
                final String skey = Arrays.toString(key);
                final String svalue = fromBytes(value);
                idwidth = (short) key.length;
                final String name = uids.id2name.put(skey, svalue);
                if (name != null) {
                  uids.error(kv, "Duplicate reverse " + kind + "  mapping: "
                             + svalue + " -> " + name
                             + " and " + svalue + " -> " + skey);
                }
              }
              if (uids.width == 0) {
                uids.width = idwidth;
              } else if (uids.width != idwidth) {
                uids.error(kv, "Invalid " + kind + " ID of length " + idwidth
                           + " (expected: " + uids.width + ')');
              }
            }
          }
        }
      }
    } catch (HBaseException e) {
      LOG.error("Error while scanning HBase, scanner=" + scanner, e);
      throw e;
    } catch (Exception e) {
      LOG.error("WTF?  Unexpected exception type, scanner=" + scanner, e);
      throw new AssertionError("Should never happen");
    }

    // Match up all forward mappings with their reverse mappings and vice
    // versa and make sure they agree.
    int errors = 0;
    for (final Map.Entry<String, Uids> entry : name2uids.entrySet()) {
      final String kind = entry.getKey();
      final Uids uids = entry.getValue();

      // Look for forward mappings without the corresponding reverse mappings.
      // These are harmful and shouldn't exist.
      for (final Map.Entry<String, String> nameid : uids.name2id.entrySet()) {
        final String name = nameid.getKey();
        final String id = nameid.getValue();
        final String found = uids.id2name.get(id);
        if (found == null) {
          uids.error("Forward " + kind + " mapping is missing reverse"
                     + " mapping: " + name + " -> " + id);
        } else if (!found.equals(name)) {
          uids.error("Forward " + kind + " mapping " + name + " -> " + id
                     + " is different than reverse mapping: "
                     + id + " -> " + found);
          final String id2 = uids.name2id.get(found);
          if (id2 != null) {
            uids.error("Inconsistent forward " + kind + " mapping "
                       + name + " -> " + id
                       + " vs " + name + " -> " + found
                       + " / " + found + " -> " + id2);
          } else {
            uids.error("Duplicate forward " + kind + " mapping "
                       + name + " -> " + id
                       + " and " + id2 + " -> " + found);
          }
        }
      }

      // Look for reverse mappings without the corresponding forward mappings.
      // These are harmless but shouldn't frequently occur.
      for (final Map.Entry<String, String> idname : uids.id2name.entrySet()) {
        final String name = idname.getValue();
        final String id = idname.getKey();
        final String found = uids.name2id.get(name);
        if (found == null) {
          LOG.warn("Reverse " + kind + " mapping is missing forward"
                   + " mapping: " + name + " -> " + id);
        } else if (!found.equals(id)) {
          final String name2 = uids.id2name.get(found);
          if (name2 != null) {
            uids.error("Inconsistent reverse " + kind + " mapping "
                       + id + " -> " + name
                       + " vs " + found + " -> " + name
                       + " / " + name2 + " -> " + found);
          } else {
            uids.error("Duplicate reverse " + kind + " mapping "
                       + id + " -> " + name
                       + " and " + found + " -> " + name2);
          }
        }
      }

      final int maxsize = Math.max(uids.id2name.size(), uids.name2id.size());
      if (uids.maxid > maxsize) {
        LOG.warn("Max ID for " + kind + " is " + uids.maxid + " but only "
                 + maxsize + " entries were found.  Maybe "
                 + (uids.maxid - maxsize) + " IDs were deleted?");
      } else if (uids.maxid < maxsize) {
        uids.error("We found " + maxsize + ' ' + kind + " but the max ID is"
                   + " only " + uids.maxid + "!  Future IDs may be"
                   + " double-assigned!");
      }

      if (uids.errors > 0) {
        LOG.error(kind + ": Found " + uids.errors + " errors.");
        errors += uids.errors;
      }
    }
    final long timing = (System.nanoTime() - start_time) / 1000000;
    System.out.println(kvcount + " KVs analyzed in " + timing + "ms (~"
                       + (kvcount * 1000 / timing) + " KV/s)");
    if (errors == 0) {
      System.out.println("No errors found.");
      return 0;
    }
    System.err.println(errors + " errors found.");
    return 1;
  }

  /**
   * Looks up an ID and finds the corresponding name(s), if any.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param lid The ID to look for.
   * @param kind The 'kind' of the ID (can be {@code null}).
   * @return The exit status of the command (0 means at least 1 found).
   */
  private static int lookupId(final HBaseClient client,
                              final byte[] table,
                              final short idwidth,
                              final long lid,
                              final String kind) {
    final byte[] id = idInBytes(idwidth, lid);
    if (id == null) {
      return 1;
    } else if (kind != null) {
      return extactLookupId(client, table, idwidth, kind, id);
    }
    return findAndPrintRow(client, table, id, NAME_FAMILY, false);
  }

  /**
   * Gets a given row in HBase and prints it on standard output.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param key The row key to attempt to get from HBase.
   * @param family The family in which we're interested.
   * @return 0 if at least one cell was found and printed, 1 otherwise.
   */
  private static int findAndPrintRow(final HBaseClient client,
                                     final byte[] table,
                                     final byte[] key,
                                     final byte[] family,
                                     boolean formard) {
    final GetRequest get = new GetRequest(table, key);
    get.family(family);
    ArrayList<KeyValue> row;
    try {
      row = client.get(get).joinUninterruptibly();
    } catch (HBaseException e) {
      LOG.error("Get failed: " + get, e);
      return 1;
    } catch (Exception e) {
      LOG.error("WTF?  Unexpected exception type, get=" + get, e);
      return 42;
    }
    return printResult(row, family, formard) ? 0 : 1;
  }

  /**
   * Looks up an ID for a given kind, and prints it if found.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param kind The 'kind' of the ID (must not be {@code null}).
   * @param id The ID to look for.
   * @return 0 if the ID for this kind was found, 1 otherwise.
   */
  private static int extactLookupId(final HBaseClient client,
                                    final byte[] table,
                                    final short idwidth,
                                    final String kind,
                                    final byte[] id) {
    final UniqueId uid = new UniqueId(client, table, kind, (int) idwidth);
    try {
      final String name = uid.getName(id);
      System.out.println(kind + ' ' + name + ": " + Arrays.toString(id));
      return 0;
    } catch (NoSuchUniqueId e) {
      LOG.error(e.getMessage());
      return 1;
    }
  }

  /**
   * Transforms an ID into the corresponding byte array.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param lid The ID to transform.
   * @return The ID represented in {@code idwidth} bytes, or
   * {@code null} if {@code lid} couldn't fit in {@code idwidth} bytes.
   */
  private static byte[] idInBytes(final short idwidth, final long lid) {
    if (idwidth <= 0) {
      throw new AssertionError("negative idwidth: " + idwidth);
    }
    final byte[] id = Bytes.fromLong(lid);
    for (int i = 0; i < id.length - idwidth; i++) {
      if (id[i] != 0) {
        System.err.println(lid + " is too large to fit on " + idwidth
            + " bytes.  Maybe you forgot to adjust --idwidth?");
        return null;
      }
    }
    return Arrays.copyOfRange(id, id.length - idwidth, id.length);
  }

  /**
   * Looks up a name and finds the corresponding UID(s), if any.
   * @param client The HBase client to use.
   * @param table The name of the HBase table to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param name The name to look for.
   * @param kind The 'kind' of the ID (can be {@code null}).
   * @return The exit status of the command (0 means at least 1 found).
   */
  private static int lookupName(final HBaseClient client,
                                final byte[] table,
                                final short idwidth,
                                final String name,
                                final String kind) {
    if (kind != null) {
      return extactLookupName(client, table, idwidth, kind, name);
    }
    return findAndPrintRow(client, table, toBytes(name), ID_FAMILY, true);
  }

  /**
   * Looks up a name for a given kind, and prints it if found.
   * @param client The HBase client to use.
   * @param idwidth Number of bytes on which the UIDs should be.
   * @param kind The 'kind' of the ID (must not be {@code null}).
   * @param name The name to look for.
   * @return 0 if the name for this kind was found, 1 otherwise.
   */
  private static int extactLookupName(final HBaseClient client,
                                      final byte[] table,
                                      final short idwidth,
                                      final String kind,
                                      final String name) {
    final UniqueId uid = new UniqueId(client, table, kind, (int) idwidth);
    try {
      final byte[] id = uid.getId(name);
      System.out.println(kind + ' ' + name + ": " + Arrays.toString(id));
      return 0;
    } catch (NoSuchUniqueName e) {
      LOG.error(e.getMessage());
      return 1;
    }
  }

  /**
   * Runs through the entire data table and creates TSMeta objects for unique
   * timeseries and/or updates {@code created} timestamps
   * The process is as follows:
   * <ul><li>Fetch the max number of Metric UIDs as we'll use those to match
   * on the data rows</li>
   * <li>Split the # of UIDs amongst worker threads</li>
   * <li>Setup a scanner in each thread for the range it will be working on and
   * start iterating</li>
   * <li>Fetch the TSUID from the row key</li>
   * <li>For each unprocessed TSUID:
   * <ul><li>Check if the metric UID mapping is present, if not, log an error
   * and continue</li>
   * <li>See if the meta for the metric UID exists, if not, create it</li>
   * <li>See if the row timestamp is less than the metric UID meta's created
   * time. This means we have a record of the UID being used earlier than the
   * meta data indicates. Update it.</li>
   * <li>Repeat the previous three steps for each of the TAGK and TAGV tags</li>
   * <li>Check to see if meta data exists for the timeseries</li>
   * <li>If not, create the counter column if it's missing, and create the meta
   * column</li>
   * <li>If it did exist, check the {@code created} timestamp and if the row's
   * time is less, update the meta data</li></ul></li>
   * <li>Continue on to the next unprocessed timeseries data row</li></ul>
   * <b>Note:</b> Updates or new entries will also be sent to the search plugin
   * if configured.
   * @param tsdb The tsdb to use for processing, including a search plugin
   * @return 0 if completed successfully, something else if it dies
   */
  private static int metaSync(final TSDB tsdb) throws Exception {
    final long start_time = System.currentTimeMillis() / 1000;
    final long max_id = getMaxMetricID(tsdb);
   
    // now figure out how many IDs to divy up between the workers
    final int workers = Runtime.getRuntime().availableProcessors() * 2;
    final double quotient = (double)max_id / (double)workers;
    final Set<Integer> processed_tsuids =
      Collections.synchronizedSet(new HashSet<Integer>());
    final ConcurrentHashMap<String, Long> metric_uids =
      new ConcurrentHashMap<String, Long>();
    final ConcurrentHashMap<String, Long> tagk_uids =
      new ConcurrentHashMap<String, Long>();
    final ConcurrentHashMap<String, Long> tagv_uids =
      new ConcurrentHashMap<String, Long>();
   
    long index = 1;
   
    LOG.info("Max metric ID is [" + max_id + "]");
    LOG.info("Spooling up [" + workers + "] worker threads");
    final Thread[] threads = new Thread[workers];
    for (int i = 0; i < workers; i++) {
      threads[i] = new MetaSync(tsdb, index, quotient, processed_tsuids,
          metric_uids, tagk_uids, tagv_uids, i);
      threads[i].setName("MetaSync # " + i);
      threads[i].start();
      index += quotient;
      if (index < max_id) {
        index++;
      }
    }
   
    // wait till we're all done
    for (int i = 0; i < workers; i++) {
      threads[i].join();
      LOG.info("[" + i + "] Finished");
    }
   
    // make sure buffered data is flushed to storage before exiting
    tsdb.flush().joinUninterruptibly();
   
    final long duration = (System.currentTimeMillis() / 1000) - start_time;
    LOG.info("Completed meta data synchronization in [" +
        duration + "] seconds");
    return 0;
  }
 
  /**
   * Runs through the tsdb-uid table and removes TSMeta, UIDMeta and TSUID
   * counter entries from the table
   * The process is as follows:
   * <ul><li>Fetch the max number of Metric UIDs</li>
   * <li>Split the # of UIDs amongst worker threads</li>
   * <li>Create a delete request with the qualifiers of any matching meta data
   * columns</li></ul>
   * <li>Continue on to the next unprocessed timeseries data row</li></ul>
   * @param tsdb The tsdb to use for processing, including a search plugin
   * @return 0 if completed successfully, something else if it dies
   */
  private static int metaPurge(final TSDB tsdb) throws Exception {
    final long start_time = System.currentTimeMillis() / 1000;
    final long max_id = getMaxMetricID(tsdb);
   
    // now figure out how many IDs to divy up between the workers
    final int workers = Runtime.getRuntime().availableProcessors() * 2;
    final double quotient = (double)max_id / (double)workers;
   
    long index = 1;
   
    LOG.info("Max metric ID is [" + max_id + "]");
    LOG.info("Spooling up [" + workers + "] worker threads");
    final Thread[] threads = new Thread[workers];
    for (int i = 0; i < workers; i++) {
      threads[i] = new MetaPurge(tsdb, index, quotient, i);
      threads[i].setName("MetaSync # " + i);
      threads[i].start();
      index += quotient;
      if (index < max_id) {
        index++;
      }
    }
   
    // wait till we're all done
    for (int i = 0; i < workers; i++) {
      threads[i].join();
      LOG.info("[" + i + "] Finished");
    }
   
    // make sure buffered data is flushed to storage before exiting
    tsdb.flush().joinUninterruptibly();
   
    final long duration = (System.currentTimeMillis() / 1000) - start_time;
    LOG.info("Completed meta data synchronization in [" +
        duration + "] seconds");
    return 0;
  }
 
  /**
   * Runs through all TSMeta objects in the UID table and passes them through
   * each of the Trees configured in the system.
   * First, the method loads all trees in the system, compiles them into
   * TreeBuilders, then scans the UID table, passing each TSMeta through each
   * of the TreeBuilder objects.
   * @param tsdb The TSDB to use for access
   * @return 0 if completed successfully, something else if an error occurred
   */
  private static int treeSync(final TSDB tsdb) throws Exception {
    final long start_time = System.currentTimeMillis() / 1000;
    final long max_id = getMaxMetricID(tsdb);
   
// now figure out how many IDs to divy up between the workers
    final int workers = Runtime.getRuntime().availableProcessors() * 2;
    final double quotient = (double)max_id / (double)workers;
   
    long index = 1;
   
    LOG.info("Max metric ID is [" + max_id + "]");
    LOG.info("Spooling up [" + workers + "] worker threads");
    final Thread[] threads = new Thread[workers];
    for (int i = 0; i < workers; i++) {
      threads[i] = new TreeSync(tsdb, index, quotient, i);
      threads[i].setName("TreeSync # " + i);
      threads[i].start();
      index += quotient;
      if (index < max_id) {
        index++;
      }
    }
   
    // wait till we're all done
    for (int i = 0; i < workers; i++) {
      threads[i].join();
      LOG.info("[" + i + "] Finished");
    }
   
    // make sure buffered data is flushed to storage before exiting
    tsdb.flush().joinUninterruptibly();
   
    final long duration = (System.currentTimeMillis() / 1000) - start_time;
    LOG.info("Completed meta data synchronization in [" +
        duration + "] seconds");
    return 0;
  }
 
  /**
   * Attempts to delete the branches, leaves, collisions and not-matched entries
   * for a given tree. Optionally removes the tree definition itself
   * @param tsdb The TSDB to use for access
   * @param tree_id ID of the tree to delete
   * @param delete_definition Whether or not to delete the tree definition as
   * well
   * @return 0 if completed successfully, something else if an error occurred
   */
  private static int purgeTree(final TSDB tsdb, final int tree_id,
      final boolean delete_definition) throws Exception {
    final TreeSync sync = new TreeSync(tsdb, 0, 1, 0);
    return sync.purgeTree(tree_id, delete_definition);
  }
 
  /**
   * Returns the max metric ID from the UID table
   * @param tsdb The TSDB to use for data access
   * @return The max metric ID as an integer value
   */
  private static long getMaxMetricID(final TSDB tsdb) {
    // first up, we need the max metric ID so we can split up the data table
    // amongst threads.
    final GetRequest get = new GetRequest(tsdb.uidTable(), new byte[] { 0 });
    get.family("id".getBytes(CHARSET));
    get.qualifier("metrics".getBytes(CHARSET));
    ArrayList<KeyValue> row;
    try {
      row = tsdb.getClient().get(get).joinUninterruptibly();
      if (row == null || row.isEmpty()) {
        throw new IllegalStateException("No data in the metric max UID cell");
      }
      final byte[] id_bytes = row.get(0).value();
      if (id_bytes.length != 8) {
        throw new IllegalStateException("Invalid metric max UID, wrong # of bytes");
      }
      return Bytes.getLong(id_bytes);
    } catch (Exception e) {
      throw new RuntimeException("Shouldn't be here", e);
    }
  }
 
  private static byte[] toBytes(final String s) {
    try {
      return (byte[]) toBytes.invoke(null, s);
    } catch (Exception e) {
      throw new RuntimeException("toBytes=" + toBytes, e);
    }
  }

  private static String fromBytes(final byte[] b) {
    try {
      return (String) fromBytes.invoke(null, b);
    } catch (Exception e) {
      throw new RuntimeException("fromBytes=" + fromBytes, e);
    }
  }
}
TOP

Related Classes of net.opentsdb.tools.UidManager

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.