// This file is part of OpenTSDB.
// Copyright (C) 2013 The OpenTSDB Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or (at your
// option) any later version. This program is distributed in the hope that it
// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details. You should have received a copy
// of the GNU Lesser General Public License along with this program. If not,
// see <http://www.gnu.org/licenses/>.
package net.opentsdb.tree;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.hbase.async.Bytes;
import org.hbase.async.GetRequest;
import org.hbase.async.HBaseException;
import org.hbase.async.KeyValue;
import org.hbase.async.PutRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.stumbleupon.async.Callback;
import com.stumbleupon.async.Deferred;
import net.opentsdb.core.TSDB;
import net.opentsdb.uid.UniqueId;
import net.opentsdb.uid.UniqueId.UniqueIdType;
import net.opentsdb.utils.JSON;
import net.opentsdb.utils.JSONException;
/**
* A leaf in a tree. Each leaf is composed, primarily, of a display name and a
* TSUID. When stored, only the display name and TSUID are recorded. When
* accessed via an RPC call, the leaf should include the metric and tags.
* <p>
* Leaves are stored as individual columns in the same row as a branch. When a
* branch is loaded with leaves, each leaf is parsed and optionally the UID
* names are loaded from the TSD. Leaf columns are stored with the column
* qualifier: "leaf:<display_name.hashCode()>". When a leaf is written to
* storage, a CompareAndSet is executed with a null value expected for the
* compare. If the compare returns false, we load the leaf at that location and
* determine if it's the same leaf. If so, it's all good and we ignore the put.
* If the TSUID is different, we record a collision in the tree so that the user
* knows their rule set matched a timeseries that was already recorded.
* @since 2.0
*/
public final class Leaf implements Comparable<Leaf> {
private static final Logger LOG = LoggerFactory.getLogger(Leaf.class);
/** Charset used to convert Strings to byte arrays and back. */
private static final Charset CHARSET = Charset.forName("ISO-8859-1");
/** ASCII Leaf prefix */
private static final byte[] LEAF_PREFIX = "leaf:".getBytes(CHARSET);
/** The metric associated with this TSUID */
private String metric = "";
/** The tags associated with this TSUID for API response purposes */
private HashMap<String, String> tags = null;
/** Display name for the leaf */
private String display_name = "";
/** TSUID the leaf links to */
private String tsuid = "";
/**
* Default empty constructor necessary for des/serialization
*/
public Leaf() {
}
/**
* Optional constructor used when building a tree
* @param display_name The name of the leaf
* @param tsuid The TSUID of the leaf
*/
public Leaf(final String display_name, final String tsuid) {
this.display_name = display_name;
this.tsuid = tsuid;
}
/** @return Hash code of the display name field */
@Override
public int hashCode() {
return display_name.hashCode();
}
/**
* Just compares the TSUID of the two objects as we don't care about the rest
* @param obj The object to compare this to
* @return True if the TSUIDs are the same or the incoming object has the same
* address
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (this.getClass() != obj.getClass()) {
return false;
}
if (obj == this) {
return true;
}
final Leaf leaf = (Leaf)obj;
return tsuid.equals(leaf.tsuid);
}
/**
* Sorts on the {@code display_name} alphabetically
* @param leaf The leaf to compare against
* @return string comparison
*/
@Override
public int compareTo(Leaf leaf) {
return display_name.compareToIgnoreCase(leaf.display_name);
}
/** @return A string describing this object */
@Override
public String toString() {
return "name: " + display_name + " tsuid: " + tsuid;
}
/**
* Calculates the column qualifier for this leaf. The qualifier is of the
* format: "leaf:<display_name.hashCode()>"
* @return The qualifier as a byte array
* @throws IllegalArgumentException if the {@code display_name} hasn't been
* set yet
*/
public byte[] columnQualifier() {
if (display_name == null || display_name.isEmpty()) {
throw new IllegalArgumentException("Missing display name");
}
final byte[] qualifier = new byte[LEAF_PREFIX.length + 4];
System.arraycopy(LEAF_PREFIX, 0, qualifier, 0, LEAF_PREFIX.length);
System.arraycopy(Bytes.fromInt(hashCode()), 0, qualifier,
LEAF_PREFIX.length, 4);
return qualifier;
}
/**
* Attempts to write the leaf to storage using a CompareAndSet call. We expect
* the stored value to be null. If it's not, we fetched the stored leaf. If
* the stored value is the TSUID as the local leaf, we return true since the
* caller is probably reprocessing a timeseries. If the stored TSUID is
* different, we store a collision in the tree and return false.
* <b>Note:</b> You MUST write the tree to storage after calling this as there
* may be a new collision. Check the tree's collision set.
* @param tsdb The TSDB to use for storage access
* @param branch_id ID of the branch this leaf belongs to
* @param tree Tree the leaf and branch belong to
* @return True if the leaf was stored successful or already existed, false
* if there was a collision
* @throws HBaseException if there was an issue
* @throws JSONException if the object could not be serialized
*/
public Deferred<Boolean> storeLeaf(final TSDB tsdb, final byte[] branch_id,
final Tree tree) {
/**
* Callback executed with the results of our CAS operation. If the put was
* successful, we just return. Otherwise we load the existing leaf to
* determine if there was a collision.
*/
final class LeafStoreCB implements Callback<Deferred<Boolean>, Boolean> {
final Leaf local_leaf;
public LeafStoreCB(final Leaf local_leaf) {
this.local_leaf = local_leaf;
}
/**
* @return True if the put was successful or the leaf existed, false if
* there was a collision
*/
@Override
public Deferred<Boolean> call(final Boolean success) throws Exception {
if (success) {
return Deferred.fromResult(success);
}
/**
* Called after fetching the existing leaf from storage
*/
final class LeafFetchCB implements Callback<Deferred<Boolean>, Leaf> {
/**
* @return True if the put was successful or the leaf existed, false if
* there was a collision
*/
@Override
public Deferred<Boolean> call(final Leaf existing_leaf)
throws Exception {
if (existing_leaf == null) {
LOG.error(
"Returned leaf was null, stored data may be corrupt for leaf: "
+ Branch.idToString(columnQualifier()) + " on branch: "
+ Branch.idToString(branch_id));
return Deferred.fromResult(false);
}
if (existing_leaf.tsuid.equals(tsuid)) {
LOG.debug("Leaf already exists: " + local_leaf);
return Deferred.fromResult(true);
}
tree.addCollision(tsuid, existing_leaf.tsuid);
LOG.warn("Branch ID: [" + Branch.idToString(branch_id)
+ "] Leaf collision with [" + tsuid +
"] on existing leaf [" + existing_leaf.tsuid +
"] named [" + display_name + "]");
return Deferred.fromResult(false);
}
}
// fetch the existing leaf so we can compare it to determine if we have
// a collision or an existing leaf
return Leaf.getFromStorage(tsdb, branch_id, display_name)
.addCallbackDeferring(new LeafFetchCB());
}
}
// execute the CAS call to start the callback chain
final PutRequest put = new PutRequest(tsdb.treeTable(), branch_id,
Tree.TREE_FAMILY(), columnQualifier(), toStorageJson());
return tsdb.getClient().compareAndSet(put, new byte[0])
.addCallbackDeferring(new LeafStoreCB(this));
}
/**
* Attempts to parse the leaf from the given column, optionally loading the
* UID names. This is used by the branch loader when scanning an entire row.
* <b>Note:</b> The column better have a qualifier that starts with "leaf:" or
* we're likely to throw a parsing exception.
* @param tsdb The TSDB to use for storage access
* @param column Column to parse a leaf from
* @param load_uids Whether or not to load UID names from the TSD
* @return The parsed leaf if successful
* @throws IllegalArgumentException if the column was missing data
* @throws NoSuchUniqueId If any of the UID name mappings do not exist
* @throws HBaseException if there was an issue
* @throws JSONException if the object could not be serialized
*/
public static Deferred<Leaf> parseFromStorage(final TSDB tsdb,
final KeyValue column, final boolean load_uids) {
if (column.value() == null) {
throw new IllegalArgumentException("Leaf column value was null");
}
// qualifier has the TSUID in the format "leaf:<display_name.hashCode()>"
// and we should only be here if the qualifier matched on "leaf:"
final Leaf leaf = JSON.parseToObject(column.value(), Leaf.class);
// if there was an error with the data and the tsuid is missing, dump it
if (leaf.tsuid == null || leaf.tsuid.isEmpty()) {
LOG.warn("Invalid leaf object in row: " + Branch.idToString(column.key()));
return Deferred.fromResult(null);
}
// if we don't need to load UIDs, then return now
if (!load_uids) {
return Deferred.fromResult(leaf);
}
// split the TSUID to get the tags
final List<byte[]> parsed_tags = UniqueId.getTagPairsFromTSUID(leaf.tsuid,
TSDB.metrics_width(), TSDB.tagk_width(), TSDB.tagv_width());
// initialize the with empty objects, otherwise the "set" operations in
// the callback won't work.
final ArrayList<String> tags = new ArrayList<String>(parsed_tags.size());
for (int i = 0; i < parsed_tags.size(); i++) {
tags.add("");
}
// setup an array of deferreds to wait on so we can return the leaf only
// after all of the name fetches have completed
final ArrayList<Deferred<Object>> uid_group =
new ArrayList<Deferred<Object>>(parsed_tags.size() + 1);
/**
* Callback executed after the UID name has been retrieved successfully.
* The {@code index} determines where the result is stored: -1 means metric,
* >= 0 means tag
*/
final class UIDNameCB implements Callback<Object, String> {
final int index;
public UIDNameCB(final int index) {
this.index = index;
}
@Override
public Object call(final String name) throws Exception {
if (index < 0) {
leaf.metric = name;
} else {
tags.set(index, name);
}
return null;
}
}
// fetch the metric name first
final byte[] metric_uid = UniqueId.stringToUid(
leaf.tsuid.substring(0, TSDB.metrics_width() * 2));
uid_group.add(tsdb.getUidName(UniqueIdType.METRIC, metric_uid).addCallback(
new UIDNameCB(-1)));
int idx = 0;
for (byte[] tag : parsed_tags) {
if (idx % 2 == 0) {
uid_group.add(tsdb.getUidName(UniqueIdType.TAGK, tag)
.addCallback(new UIDNameCB(idx)));
} else {
uid_group.add(tsdb.getUidName(UniqueIdType.TAGV, tag)
.addCallback(new UIDNameCB(idx)));
}
idx++;
}
/**
* Called after all of the UID name fetches have completed and parses the
* tag name/value list into name/value pairs for proper display
*/
final class CollateUIDsCB implements Callback<Deferred<Leaf>,
ArrayList<Object>> {
/**
* @return A valid Leaf object loaded with UID names
*/
@Override
public Deferred<Leaf> call(final ArrayList<Object> name_calls)
throws Exception {
int idx = 0;
String tagk = "";
leaf.tags = new HashMap<String, String>(tags.size() / 2);
for (String name : tags) {
if (idx % 2 == 0) {
tagk = name;
} else {
leaf.tags.put(tagk, name);
}
idx++;
}
return Deferred.fromResult(leaf);
}
}
// wait for all of the UID name fetches in the group to complete before
// returning the leaf
return Deferred.group(uid_group).addCallbackDeferring(new CollateUIDsCB());
}
/** @return The configured leaf column prefix */
public static byte[] LEAF_PREFIX() {
return LEAF_PREFIX;
}
/**
* Writes the leaf to a JSON object for storage. This is necessary for the CAS
* calls and to reduce storage costs since we don't need to store UID names
* (particularly as someone may rename a UID)
* @return The byte array to store
*/
private byte[] toStorageJson() {
final ByteArrayOutputStream output = new ByteArrayOutputStream(
display_name.length() + tsuid.length() + 30);
try {
final JsonGenerator json = JSON.getFactory().createGenerator(output);
json.writeStartObject();
// we only need to write a small amount of information
json.writeObjectField("displayName", display_name);
json.writeObjectField("tsuid", tsuid);
json.writeEndObject();
json.close();
// TODO zero copy?
return output.toByteArray();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Attempts to fetch the requested leaf from storage.
* <b>Note:</b> This method will not load the UID names from a TSDB. This is
* only used to fetch a particular leaf from storage for collision detection
* @param tsdb The TSDB to use for storage access
* @param branch_id ID of the branch this leaf belongs to
* @param display_name Name of the leaf
* @return A valid leaf if found, null if the leaf did not exist
* @throws HBaseException if there was an issue
* @throws JSONException if the object could not be serialized
*/
private static Deferred<Leaf> getFromStorage(final TSDB tsdb,
final byte[] branch_id, final String display_name) {
final Leaf leaf = new Leaf();
leaf.setDisplayName(display_name);
final GetRequest get = new GetRequest(tsdb.treeTable(), branch_id);
get.family(Tree.TREE_FAMILY());
get.qualifier(leaf.columnQualifier());
/**
* Called with the results of the fetch from storage
*/
final class GetCB implements Callback<Deferred<Leaf>, ArrayList<KeyValue>> {
/**
* @return null if the row was empty, a valid Leaf if parsing was
* successful
*/
@Override
public Deferred<Leaf> call(ArrayList<KeyValue> row) throws Exception {
if (row == null || row.isEmpty()) {
return Deferred.fromResult(null);
}
final Leaf leaf = JSON.parseToObject(row.get(0).value(), Leaf.class);
return Deferred.fromResult(leaf);
}
}
return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}
// GETTERS AND SETTERS ----------------------------
/** @return The metric associated with this TSUID */
public String getMetric() {
return metric;
}
/** @return The tags associated with this TSUID */
public Map<String, String> getTags() {
return tags;
}
/** @return The public name of this leaf */
public String getDisplayName() {
return display_name;
}
/** @return the tsuid */
public String getTsuid() {
return tsuid;
}
/** @param metric The metric associated with this TSUID */
public void setMetric(final String metric) {
this.metric = metric;
}
/** @param tags The tags associated with this TSUID */
public void setTags(final HashMap<String, String> tags) {
this.tags = tags;
}
/** @param display_name Public display name for the leaf */
public void setDisplayName(final String display_name) {
this.display_name = display_name;
}
/** @param tsuid the tsuid to set */
public void setTsuid(final String tsuid) {
this.tsuid = tsuid;
}
}