* @param idx
* @throws Exception
*/
protected void traverse(final StateVertex parent, final int idx) throws Exception {
assert (idx < this.num_elements && idx >= 0);
final CatalogType current = this.all_search_elements.get(idx);
assert (current != null);
final String current_key = this.all_search_keys.get(idx);
this.traverse_ctr++;
final boolean is_table = (idx < this.num_tables);
// Are we the last element in the list, thereby creating a complete
// solution
final boolean complete_solution = (idx + 1 == this.num_elements);
if (is_table)
assert (current instanceof Table);
final String spacer = BranchAndBoundPartitioner.TRAVERSAL_SPACERS.get(idx);
if (this.halt_search == false) {
assert (this.halt_reason == null);
if (hints.limit_back_tracks != null && hints.limit_back_tracks >= 0 && this.backtrack_ctr > hints.limit_back_tracks) {
LOG.info("Hit back track limit. Halting search [" + this.backtrack_ctr + "]");
this.halt_search = true;
this.halt_reason = HaltReason.BACKTRACK_LIMIT;
return;
} else if (this.halt_time != null && System.currentTimeMillis() >= this.halt_time) {
LOG.info("Hit time limit. Halting search [" + this.backtrack_ctr + "]");
this.halt_search = true;
this.halt_reason = (this.halt_time_local ? HaltReason.LOCAL_TIME_LIMIT : HaltReason.GLOBAL_TIME_LIMIT);
return;
}
} else
return;
// Get the list of possible attributes that we could use for this
// current element
Collection<CatalogType> current_attributes = null;
try {
current_attributes = this.cp.getCandidateValues(current, CatalogType.class);
} catch (IllegalArgumentException ex) {
throw new RuntimeException("Failed to get candidate attributes for " + current, ex);
}
final int num_attributes = current_attributes.size();
assert (current_attributes != null);
if (trace.val) {
Map<String, Object> m = new LinkedHashMap<String, Object>();
m.put("Traversal Level", traverse_ctr);
m.put("Item Index", idx);
m.put("Current", current);
m.put("Attributes", String.format("COUNT=%d\n%s", num_attributes, StringUtil.join("\n", CatalogUtil.getDisplayNames(current_attributes))));
LOG.trace(StringUtil.formatMaps(m));
}
// Keep track of whether we have a VerticalPartitionColumn that
// needs to be
// reset after each round
VerticalPartitionColumn vp_col = null;
// Get our workload filter for this level of the traversal
Filter filter = BranchAndBoundPartitioner.this.traversal_filters.get(current);
// Descendant tables used for memory calculations
// It's ok for it to be empty. That means we're searching against
// all of tables
Set<Table> current_previousTables = this.previous_tables[idx];
assert (current_previousTables != null) : String.format("No previous tables at index %d? [current=%s, num_tables=%d]", idx, current, num_tables);
// The local best vertex is the best vertex for this level in the
// traversal
StateVertex local_best_vertex = null;
// Just make sure that if we have a VerticalPartitionColumn in our
// list of
// current attributes that it's optimizations are not applied
// If they are, then we can disable them
for (CatalogType attribute : current_attributes) {
if (attribute instanceof VerticalPartitionColumn && ((VerticalPartitionColumn) attribute).isUpdateApplied()) {
((VerticalPartitionColumn) attribute).revertUpdate();
}
} // FOR
// Iterate through the columns and find the one with the best cost
int attribute_ctr = 0;
for (CatalogType attribute : current_attributes) {
assert (attribute != null) : "Null attribute key for " + current + ": " + current_attributes;
String attribute_key = CatalogKey.createKey(attribute);
if (trace.val)
LOG.trace("Evaluating " + attribute.fullName());
boolean memory_exceeded = false;
Long memory = null;
CatalogType current_attribute = null;
// Is this the last element we have to look at
boolean last_attribute = (++attribute_ctr == num_attributes);
// Dynamic Debugging
this.cost_model.setDebuggingEnabled(this.hints.isDebuggingEnabled(attribute_key));
// IMPORTANT: We have to invalidate the cache for the current
// element *AND* all those levels
// below us in the search tree!
if (this.cost_model.isCachingEnabled()) {
for (int i = idx; i < this.num_elements; i++) {
if (trace.val)
LOG.trace("Invalidating " + this.all_search_keys.get(i));
this.cost_model.invalidateCache(this.all_search_keys.get(i));
} // FOR
// If we're not using caching, then just clear out the cost
// model completely
} else {
this.cost_model.clear();
}
// ----------------------------------------------
// TABLE PARTITIONING KEY
// ----------------------------------------------
if (is_table) {
Table current_tbl = (Table) current;
Column search_col = (Column) attribute;
Column current_col = null;
// Check whether this is our replication marker column
if (search_col instanceof ReplicatedColumn) {
current_tbl.setIsreplicated(true);
current_col = ReplicatedColumn.get(current_tbl);
}
// VerticalPartitionColumn
else if (search_col instanceof VerticalPartitionColumn) {
// We need to update the statements that can use them
// using the pre-compiled query plans
current_tbl.setIsreplicated(false);
current_col = search_col;
vp_col = (VerticalPartitionColumn) search_col;
assert (CatalogUtil.getDatabase(vp_col).equals(info.catalogContext.database)) : String.format("VP_COL[%d] != INFO[%d]", CatalogUtil.getDatabase(vp_col).hashCode(),
info.catalogContext.database.hashCode());
MaterializedViewInfo catalog_view = vp_col.applyUpdate();
assert (catalog_view != null) : "Unexpected null MaterializedViewInfo for " + current_tbl + " vertical partition:\n" + vp_col;
if (this.cost_model.isCachingEnabled()) {
if (trace.val)
LOG.trace("Invalidating VerticalPartition Statements in cost model: " + vp_col.getOptimizedQueries());
this.cost_model.invalidateCache(vp_col.getOptimizedQueries());
}
TableStatistics tstats = VerticalPartitionerUtil.computeTableStatistics(vp_col, info.stats);
assert (tstats != null);
// Add the vp's sys table to the list of tables that we
// need to estimate the memory
assert (catalog_view.getDest() != null) : "Missing parent table for " + catalog_view.fullName();
assert (this.current_vertical_partitions.contains(catalog_view.getDest()) == false) : vp_col;
this.current_vertical_partitions.add(catalog_view.getDest());
}
// MultiColumn
else if (search_col instanceof MultiColumn) {
// Nothing special?
current_tbl.setIsreplicated(false);
current_col = search_col;
}
// Otherwise partition on this particular column
else {
current_tbl.setIsreplicated(false);
current_col = current_tbl.getColumns().get(search_col.getName());
}
// We should always have a horizontal partition column
assert (current_col != null);
current_tbl.setPartitioncolumn(current_col);
assert (current_col.getName().equals(current_tbl.getPartitioncolumn().getName())) : "Unexpected " + current_col.fullName() + " != " + current_tbl.getPartitioncolumn().fullName();
// Estimate memory size
Collection<Table> tablesToEstimate = null;
if (hints.enable_vertical_partitioning && this.current_vertical_partitions.isEmpty() == false) {
tablesToEstimate = CollectionUtils.union(current_previousTables, this.current_vertical_partitions);
} else {
tablesToEstimate = current_previousTables;
}
if (trace.val)
LOG.trace(String.format("Calculating memory size of current solution [%s]:\n%s", current_col.fullName(), StringUtil.join("\n", current_previousTables)));
try {
memory = this.memory_estimator.estimate(info.catalogContext.database, info.getNumPartitions(), tablesToEstimate);
} catch (Throwable ex) {
throw new RuntimeException("Failed to estimate memory using new attribute " + current_col.fullName(), ex);
}
memory_exceeded = (memory > this.hints.max_memory_per_partition);
if (trace.val)
LOG.trace(String.format("%s Memory: %s [ratio=%.2f, exceeded=%s]", current_col.fullName(), StringUtil.formatSize(memory), memory / (double) hints.max_memory_per_partition,
memory_exceeded));
current_attribute = current_col;
// ----------------------------------------------
// PROCEDURE PARTITIONING PARAMETER
// ----------------------------------------------
} else {
Procedure current_proc = (Procedure) current;
ProcParameter current_proc_param = (ProcParameter) attribute;
assert (current_proc_param != null);
current_proc.setPartitionparameter(current_proc_param.getIndex());
memory = parent.memory;
current_attribute = current_proc_param;
}
// ----------------------------------------------
// COST ESTIMATION
// ----------------------------------------------
Double cost = null;
Double singlep_txns = null;
// Don't estimate the cost if it doesn't fit
if (!memory_exceeded) {
cost = this.cost_model.estimateWorkloadCost(info.catalogContext, info.workload, filter, best_vertex.cost);
singlep_txns = this.cost_model.getSinglePartitionProcedureHistogram().getSampleCount() / (double) this.cost_model.getProcedureHistogram().getSampleCount();
} else {
cost = Double.MAX_VALUE;
}
StateVertex state = new StateVertex(parent, current_key, attribute_key, is_table, cost, singlep_txns, memory);
if (debug.val)
state.debug = cost_model.debugHistograms(info.catalogContext);
// if (!this.graph.containsVertex(parent))
// this.graph.addVertex(parent);
// this.graph.addEdge(new StateEdge(), parent, state,
// EdgeType.DIRECTED);
if (local_best_vertex == null || local_best_vertex.cost > state.cost)
local_best_vertex = state;
assert (local_best_vertex != null);
// ----------------------------------------------
// DEBUG OUTPUT
// ----------------------------------------------
// state.debug = this.cost_model.getLastDebugMessage();
LOG.info(this.createLevelOutput(state, CatalogUtil.getDisplayName(current_attribute, false), spacer, memory_exceeded));
// ----------------------------------------------
// ANALYSIS
// ----------------------------------------------
if (memory_exceeded) {
if (debug.val)
LOG.debug("Memory exceeeded! Skipping solution!");
if (vp_col != null) {
this.revertVerticalPartitionColumn(vp_col);
vp_col = null;
}
continue;
}
// The cost of our parent can never be greater than our cost
if (!parent.isStartVertex()) {
boolean is_valid = MathUtil.greaterThanEquals(cost.floatValue(), parent.cost.floatValue(), 0.001f);
if (!is_valid) { // && debug.get()) {
Map<String, Object> m0 = new LinkedHashMap<String, Object>();
m0.put("Parent State", parent.toString());
m0.put("Parent CostModel", parent.debug);
m0.put("Parent Catalog", createPartitionPlan(hints, parent, false, false));
Map<String, Object> m1 = new LinkedHashMap<String, Object>();
m1.put("Current Attribute", current_attribute.fullName());
m1.put("Current State", state.toString());
m1.put("Current CostModel", cost_model.debugHistograms(info.catalogContext));
m1.put("Current Catalog", createPartitionPlan(hints, state, false, false));
LOG.error("CURRENT COST IS GREATER THAN CURRENT COST! THIS CANNOT HAPPEN!\n" + StringUtil.formatMaps(m0, m1));