Examples of com.facebook.presto.execution.DataSource

com.facebook.presto.execution.DataSource

    public void testNoPruning()
    {
        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of()));
        assertEquals(partitions.size(), 2);


        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, BooleanLiteral.TRUE_LITERAL, Predicates.<Partition>alwaysTrue(), ImmutableMap.<Symbol, ColumnHandle>of(new Symbol("ds"), dsColumnHandle));
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        assertEquals(splits.size(), 4);
    }

View Full Code Here

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "foo")));
        assertEquals(partitions.size(), 2);


        // ds=3. No partition will match this.
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("ds")), new StringLiteral("3"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // no splits found
        assertEquals(splits.size(), 0);
    }

View Full Code Here

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "1")));
        assertEquals(partitions.size(), 2);


        // ds=1. One partition with three splits will match this.
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("ds")), new StringLiteral("1"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // three splits found
        assertEquals(splits.size(), 3);
    }

View Full Code Here

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "foo")));
        assertEquals(partitions.size(), 2);


        // foo=bar. Not a prunable column
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("foo")), new StringLiteral("bar"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // all splits found
        assertEquals(splits.size(), 4);
    }

View Full Code Here

        List<TaskSource> sources = new ArrayList<>();
        long sequenceId = 0;
        for (PlanNode sourceNode : subplan.getFragment().getSources()) {
            TableScanNode tableScan = (TableScanNode) sourceNode;


            DataSource dataSource = splitManager.getSplits(session,
                    tableScan.getTable(),
                    tableScan.getPartitionPredicate(),
                    tableScan.getUpstreamPredicateHint(),
                    Predicates.<Partition>alwaysTrue(),
                    tableScan.getAssignments());


            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            for (Split split : dataSource.getSplits()) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
            }


            sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
        }

View Full Code Here

            List<Partition> partitions = FluentIterable.from(getPartitions(node))
                    .filter(materializedViewPartitionPredicate)
                    .toList();


            // get dataSource for table
            DataSource dataSource = splitManager.getPartitionSplits(node.getTable(), partitions);


            return new NodeSplits(node.getId(), dataSource);
        }

View Full Code Here

                    return node.getSource().accept(this, materializedViewPartitionPredicate);


                case SYSTEM: {
                    NodeSplits nodeSplits = node.getSource().accept(this, materializedViewPartitionPredicate);
                    if (nodeSplits.dataSource.isPresent()) {
                        DataSource dataSource = nodeSplits.dataSource.get();
                        final double ratio = node.getSampleRatio();
                        Iterable<Split> sampleIterable = Iterables.filter(dataSource.getSplits(), new Predicate<Split>()
                        {
                            public boolean apply(@Nullable Split input)
                            {
                                return ThreadLocalRandom.current().nextDouble() < ratio;
                            }
                        });
                        DataSource sampledDataSource = new DataSource(dataSource.getDataSourceName(), sampleIterable);


                        return new NodeSplits(node.getId(), sampledDataSource);
                    }
                    else {
                        // table sampling on a sub query without splits is meaningless

View Full Code Here

            MaterializedViewWriter materializedViewWriter = new MaterializedViewWriter(node, shardManager);


            // get source splits
            NodeSplits nodeSplits = node.getSource().accept(this, materializedViewWriter.getPartitionPredicate());
            checkState(nodeSplits.dataSource.isPresent(), "No splits present for import");
            DataSource dataSource = nodeSplits.dataSource.get();


            // record output
            outputReceivers.put(node.getId(), materializedViewWriter.getOutputReceiver());


            // wrap splits with table writer info
            Iterable<Split> newSplits = materializedViewWriter.wrapSplits(nodeSplits.planNodeId, dataSource.getSplits());
            return new NodeSplits(node.getId(), new DataSource(dataSource.getDataSourceName(), newSplits));
        }

View Full Code Here

        List<TaskSource> sources = new ArrayList<>();
        long sequenceId = 0;
        for (PlanNode sourceNode : subplan.getFragment().getSources()) {
            TableScanNode tableScan = (TableScanNode) sourceNode;


            DataSource dataSource = splitManager.getSplits(session,
                    tableScan.getTable(),
                    tableScan.getPartitionPredicate(),
                    tableScan.getUpstreamPredicateHint(),
                    Predicates.<Partition>alwaysTrue(),
                    tableScan.getAssignments());


            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            for (Split split : dataSource.getSplits()) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
            }


            sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
        }

View Full Code Here

    public void testNoPruning()
    {
        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of()));
        assertEquals(partitions.size(), 2);


        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, BooleanLiteral.TRUE_LITERAL, Predicates.<Partition>alwaysTrue(), ImmutableMap.<Symbol, ColumnHandle>of(new Symbol("ds"), dsColumnHandle));
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        assertEquals(splits.size(), 4);
    }

View Full Code Here

0 1 2

TOP

Related Classes of com.facebook.presto.execution.DataSource

com.facebook.presto.split.SplitManager

com.facebook.presto.split.TestNativeSplitManager

com.facebook.presto.sql.planner.DistributedExecutionPlanner$Visitor

com.facebook.presto.util.LocalQueryRunner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.