* ModeShape (http://www.modeshape.org)

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*       http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* A {@link QueryEngine} implementation that always scans all nodes in the workspace(s) and filtering out any node that does not

* satisfy the criteria. This scanning is not very efficient and can result in slow queries, especially when the repository is

* quite large or when the number of nodes that satisfies the query's criteria is a small fraction of all possible nodes in the

* workspace(s).

* <p>

* However, this fully-functional QueryEngine implementation is designed to be subclassed when the nodes for a particular source

* (and optionally criteria) can be found more quickly. In such cases, the subclass should override the

* {@link #createNodeSequenceForSource(QueryCommand, QueryContext, PlanNode, Columns, QuerySources)} method or the

* {@link #createNodeSequenceForSource(QueryCommand, QueryContext, PlanNode, IndexPlan, Columns, QuerySources)} method and return

* a NodeSequence that contains only the applicable nodes.

* </p>

*/

    /** We don't use the standard logging convention here; we want clients to easily configure logging for the indexes */

    protected static final Logger LOGGER = Logger.getLogger("org.modeshape.jcr.query");

    public static class Builder extends QueryEngineBuilder {

        @Override

        public QueryEngine build() {

            return new ScanningQueryEngine(context(), repositoryName(), planner(), optimizer());

        }

        @Override

        protected Optimizer defaultOptimizer() {

            return new RuleBasedOptimizer();

        }

    }

    public static Builder builder() {

        return new Builder();

    }

    protected final String repositoryName;

    protected final Planner planner;

    protected final Optimizer optimizer;

    public ScanningQueryEngine( ExecutionContext context,

                                String repositoryName,

                                Planner planner,

                                Optimizer optimizer ) {

        assert planner != null;

        assert optimizer != null;

        this.repositoryName = repositoryName;

        this.planner = planner;

        this.optimizer = optimizer;

    }

    /**

     * Execute the supplied query by planning, optimizing, and then processing it.

     *

     * @param queryContext the context in which the query should be executed; same instance as returned by

     *        {@link #createQueryContext}

     * @param query the query that is to be executed

     * @return the query results; never null

     * @throws IllegalArgumentException if the context or query references are null

     * @throws QueryCancelledException if the query was cancelled

     * @throws RepositoryException if there was a problem executing the query

     */

    @Override

    public QueryResults execute( final QueryContext queryContext,

                                 final QueryCommand query ) throws QueryCancelledException, RepositoryException {

        CheckArg.isNotNull(queryContext, "queryContext");

        CheckArg.isNotNull(query, "query");

        final ScanQueryContext context = (ScanQueryContext)queryContext;

        checkCancelled(context);

        // Validate that all of the referenced variables have been provided ...

        Visitors.visitAll(query, new Visitors.AbstractVisitor() {

            @Override

            public void visit( BindVariableName obj ) {

                if (!context.getVariables().keySet().contains(obj.getBindVariableName())) {

                    context.getProblems().addError(GraphI18n.missingVariableValue, obj.getBindVariableName());

                }

            }

        });

        boolean trace = LOGGER.isTraceEnabled();

        if (trace) {

            LOGGER.trace("Beginning to process query {3} against workspace(s) {0} in '{1}' repository: {2}",

                         context.getWorkspaceNames(), repositoryName, query, context.id());

        }

        // Create the canonical plan ...

        long start = System.nanoTime();

        PlanNode plan = planner.createPlan(context, query);

        long duration = Math.abs(System.nanoTime() - start);

        Statistics stats = new Statistics(duration);

        final String workspaceName = context.getWorkspaceNames().iterator().next();

        if (trace) {

            LOGGER.trace("Computed canonical query plan for query {0}: {1}", context.id(), plan);

        }

        checkCancelled(context);

        Columns resultColumns = null;

        if (!context.getProblems().hasErrors()) {

            // Optimize the plan ...

            start = System.nanoTime();

            PlanNode optimizedPlan = optimizer.optimize(context, plan);

            duration = Math.abs(System.nanoTime() - start);

            stats = stats.withOptimizationTime(duration);

            if (trace) {

                LOGGER.trace("Computed optimized query plan for query {0}:\n{1}", context.id(), optimizedPlan);

            }

            // Find the query result columns ...

            start = System.nanoTime();

            // Determine the Columns object for specific nodes in the plan, and store them in the context ...

            optimizedPlan.apply(Traversal.POST_ORDER, new PlanNode.Operation() {

                @Override

                public void apply( PlanNode node ) {

                    Columns columns = null;

                    switch (node.getType()) {

                        case PROJECT:

                        case SOURCE:

                            columns = determineProjectedColumns(node, context);

                            assert columns != null;

                            break;

                        case JOIN:

                            Columns leftColumns = context.columnsFor(node.getFirstChild());

                            Columns rightColumns = context.columnsFor(node.getLastChild());

                            columns = leftColumns.with(rightColumns);

                            assert columns != null;

                            break;

                        case DEPENDENT_QUERY:

                            columns = context.columnsFor(node.getLastChild());

                            assert columns != null;

                            break;

                        case SET_OPERATION:

                            leftColumns = context.columnsFor(node.getFirstChild());

                            rightColumns = context.columnsFor(node.getLastChild());

                            if (checkUnionCompatible(leftColumns, rightColumns, context, query)) {

                                columns = leftColumns;

                                assert columns != null;

                            }

                            break;

                        case INDEX:

                            // do nothing with indexes ...

                            break;

                        default:

                            assert node.getChildCount() == 1;

                            columns = context.columnsFor(node.getFirstChild());

                            assert columns != null;

                            break;

                    }

                    if (columns != null) {

                        context.addColumnsFor(node, columns);

                    }

                }

            });

            Problems problems = context.getProblems();

            if (problems.hasErrors()) {

                throw new RepositoryException(JcrI18n.problemsWithQuery.text(query, problems.toString()));

            } else if (LOGGER.isDebugEnabled() && problems.hasWarnings()) {

                LOGGER.debug("There are several warnings with this query: {0}\n{1}", query, problems.toString());

            }

            resultColumns = context.columnsFor(optimizedPlan);

            assert resultColumns != null;

            duration = Math.abs(System.nanoTime() - start);

            stats = stats.withResultsFormulationTime(duration);

            if (trace) {

                LOGGER.trace("Computed output columns for query {0}: {1}", context.id(), resultColumns);

            }

            if (!context.getProblems().hasErrors()) {

                checkCancelled(context);

                // Execute the plan ...

                try {

                    start = System.nanoTime();

                    if (trace) {

                        LOGGER.trace("Start executing query {0}", context.id());

                    }

                    QueryResults results = executeOptimizedQuery(context, query, stats, optimizedPlan);

                    if (trace) {

                        LOGGER.trace("Stopped executing query {0}: {1}", context.id(), stats);

                    }

                    return results;

                } finally {

                    duration = Math.abs(System.nanoTime() - start);

                    stats = stats.withExecutionTime(duration);

                }

            }

        }

        // Check whether the query was cancelled during execution ...

        checkCancelled(context);

        if (resultColumns == null) resultColumns = ResultColumns.EMPTY;

        // There were problems somewhere ...

        int width = resultColumns.getColumns().size();

        CachedNodeSupplier cachedNodes = context.getNodeCache(workspaceName);

        return new Results(resultColumns, stats, NodeSequence.emptySequence(width), cachedNodes, context.getProblems(), null);

    }

    /**

     * Determine whether this column and the other are <i>union-compatible</i> (that is, having the same columns).

     *

     * @param results1 the first result set; may not be null

     * @param results2 the second result set; may not be null

     * @param context the query execution context; may not be null

     * @param query the query being executed; may not be null

     * @return true if the supplied columns definition are union-compatible, or false if they are not

     */

    protected boolean checkUnionCompatible( Columns results1,

                                            Columns results2,

                                            ScanQueryContext context,

                                            QueryCommand query ) {

        if (results1 == results2) return true;

        if (results1 == null || results2 == null) return false;

        if (results1.hasFullTextSearchScores() != results2.hasFullTextSearchScores()) {

            // The query is not compatible

            context.getProblems().addError(JcrI18n.setQueryContainsResultSetsWithDifferentFullTextSearch);

            return false;

        }

        if (results1.getColumns().size() != results2.getColumns().size()) {

            // The query is not compatible

            context.getProblems().addError(JcrI18n.setQueryContainsResultSetsWithDifferentNumberOfColumns,

                                           results1.getColumns().size(), results2.getColumns().size());

            return false;

        }

        // Go through the columns and make sure that the property names and types match ...

        // (we can't just check column names, since the column names may include the selector if more than one selector)

        int numColumns = results1.getColumns().size();

        boolean noProblems = true;

        for (int i = 0; i != numColumns; ++i) {

            Column thisColumn = results1.getColumns().get(i);

            Column thatColumn = results2.getColumns().get(i);

            if (!thisColumn.getPropertyName().equalsIgnoreCase(thatColumn.getPropertyName())) return false;

            String thisType = results1.getColumnTypeForProperty(thisColumn.getSelectorName(), thisColumn.getPropertyName());

            String thatType = results2.getColumnTypeForProperty(thatColumn.getSelectorName(), thatColumn.getPropertyName());

            if (!thisType.equalsIgnoreCase(thatType)) {

                // The query is not compatible

                context.getProblems().addError(JcrI18n.setQueryContainsResultSetsWithDifferentColumns, thisColumn, thatColumn);

                noProblems = false;

            }

        }

        return noProblems;

    }

    /**

     * Compute the columns that are defined in the supplied {@link PlanNode plan node}. If the supplied plan node is not a

     * {@link Type#PROJECT project node}, the method finds the first PROJECT node below the given node.

     *

     * @param optimizedPlan the optimized plan node in a query plan; may not be null

     * @param context the query context; may not be null

     * @return the representation of the projected columns; never null

     */

    protected Columns determineProjectedColumns( PlanNode optimizedPlan,

                                                 final ScanQueryContext context ) {

        final PlanHints hints = context.getHints();

        // Look for which columns to include in the results; this will be defined by the highest PROJECT node ...

        PlanNode project = optimizedPlan;

        if (project.getType() != Type.PROJECT) {

            project = optimizedPlan.findAtOrBelow(Traversal.LEVEL_ORDER, Type.PROJECT);

        }

        if (project != null) {

            List<Column> columns = project.getPropertyAsList(Property.PROJECT_COLUMNS, Column.class);

            List<String> columnTypes = project.getPropertyAsList(Property.PROJECT_COLUMN_TYPES, String.class);

            // Determine whether to include the full-text search scores in the results ...

            boolean includeFullTextSearchScores = hints.hasFullTextSearch;

            if (!includeFullTextSearchScores) {

                for (PlanNode select : optimizedPlan.findAllAtOrBelow(Type.SELECT)) {

                    Constraint constraint = select.getProperty(Property.SELECT_CRITERIA, Constraint.class);

                    if (QueryUtil.includeFullTextScores(constraint)) {

                        includeFullTextSearchScores = true;

                        break;

                    }

                }

            }

            // The projected columns may not include all of the selectors from the child of the PROJECT node.

            // So, we need to figure out the selector indexes based upon the ResultColumn for the child ...

            Columns childColumns = context.columnsFor(project.getFirstChild());

            return new ResultColumns(columns, columnTypes, includeFullTextSearchScores, childColumns);

        }

        // Look for a SOURCE ...

        if (optimizedPlan.getType() == Type.SOURCE) {

            PlanNode source = optimizedPlan;

            List<Schemata.Column> schemataColumns = source.getPropertyAsList(Property.SOURCE_COLUMNS, Schemata.Column.class);

            List<Column> columns = new ArrayList<>(schemataColumns.size());

            List<String> columnTypes = new ArrayList<>(schemataColumns.size());

            SelectorName selector = source.getSelectors().iterator().next();

            for (Schemata.Column schemataColumn : schemataColumns) {

                Column column = new Column(selector, schemataColumn.getName(), schemataColumn.getName());

                columns.add(column);

                columnTypes.add(schemataColumn.getPropertyTypeName());

            }

            return new ResultColumns(columns, columnTypes, hints.hasFullTextSearch, null);

        }

        return ResultColumns.EMPTY;

    }

    private void checkCancelled( QueryContext context ) throws QueryCancelledException {

        if (context.isCancelled()) {

            throw new QueryCancelledException();

        }

    }

    @Override

    public void shutdown() {

        // nothing to do

    }

    @Override

    public QueryContext createQueryContext( ExecutionContext context,

                                            RepositoryCache repositoryCache,

                                            Set<String> workspaceNames,

                                            Map<String, NodeCache> overriddenNodeCachesByWorkspaceName,

                                            Schemata schemata,

                                            RepositoryIndexes indexDefns,

                                            NodeTypes nodeTypes,

                                            BufferManager bufferManager,

                                            PlanHints hints,

                                            Map<String, Object> variables ) {

        return new ScanQueryContext(context, repositoryCache, workspaceNames, overriddenNodeCachesByWorkspaceName, schemata,

                                    indexDefns, nodeTypes, bufferManager, hints, null, variables,

                                    new HashMap<PlanNode, Columns>());

    }

    /**

     * Execute the optimized query defined by the supplied {@link PlanNode plan node}.

     *

     * @param context the context in which the query is to be executed; may not be null

     * @param command the original query; may not be null

     * @param statistics the statistics for the current query execution

     * @param plan the optimized plan for the query; may not be null

     * @return the query results; never null but possibly empty

     */

    protected QueryResults executeOptimizedQuery( final ScanQueryContext context,

                                                  QueryCommand command,

                                                  Statistics statistics,

                                                  PlanNode plan ) {

        long nanos = System.nanoTime();

        Columns columns = null;

        NodeSequence rows = null;

        final String workspaceName = context.getWorkspaceNames().iterator().next();

        try {

            // Find the topmost PROJECT node and build the Columns ...

            PlanNode project = plan.findAtOrBelow(Type.PROJECT);

            assert project != null;

            columns = context.columnsFor(plan);

            assert columns != null;

            boolean trace = LOGGER.isTraceEnabled();

            if (context.getHints().planOnly) {

                if (trace) {

                    LOGGER.trace("Request for only query plan when executing query {0}", context.id());

                }

                rows = NodeSequence.emptySequence(columns.getColumns().size());

            } else {

                boolean includeSystemContent = context.getHints().includeSystemContent;

                final QuerySources sources = new QuerySources(context.getRepositoryCache(), context.getNodeTypes(),

                                                              workspaceName, includeSystemContent);

                rows = createNodeSequence(command, context, plan, columns, sources);

                long nanos2 = System.nanoTime();

                statistics = statistics.withResultsFormulationTime(Math.abs(nanos2 - nanos));

                nanos = nanos2;

                if (rows == null) {

                    // There must have been an error or was cancelled ...

                    assert context.getProblems().hasErrors() || context.isCancelled();

                    rows = NodeSequence.emptySequence(columns.getColumns().size());

                }

                if (trace) {

                    LOGGER.trace("The execution function for {0}: {1}", context.id(), rows);

                }

            }

        } finally {

            statistics = statistics.withExecutionTime(Math.abs(System.nanoTime() - nanos));

        }

        final String planDesc = context.getHints().showPlan ? plan.getString() : null;

        CachedNodeSupplier cachedNodes = context.getNodeCache(workspaceName);

        return new Results(columns, statistics, rows, cachedNodes, context.getProblems(), planDesc);

    }

    /**

     * Create a node sequence containing the results of the original query as defined by the supplied plan.

     *

     * @param originalQuery the original query command; may not be null

     * @param context the context in which the query is to be executed; may not be null

     * @param plan the optimized plan for the query; may not be null

     * @param columns the result column definition; may not be null

     * @param sources the query sources for the repository; may not be null

     * @return the sequence of results; null only if the type of plan is not understood

     */

    protected NodeSequence createNodeSequence( QueryCommand originalQuery,

                                               ScanQueryContext context,

                                               PlanNode plan,

                                               Columns columns,

                                               QuerySources sources ) {

        NodeSequence rows = null;

        final String workspaceName = sources.getWorkspaceName();

        final NodeCache cache = context.getNodeCache(workspaceName);

        final TypeSystem types = context.getTypeSystem();

        final BufferManager bufferManager = context.getBufferManager();

        switch (plan.getType()) {