Examples of com.facebook.presto.execution.DataSource

com.facebook.presto.execution.DataSource

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "foo")));
        assertEquals(partitions.size(), 2);


        // ds=3. No partition will match this.
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("ds")), new StringLiteral("3"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // no splits found
        assertEquals(splits.size(), 0);
    }

View Full Code Here

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "1")));
        assertEquals(partitions.size(), 2);


        // ds=1. One partition with three splits will match this.
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("ds")), new StringLiteral("1"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // three splits found
        assertEquals(splits.size(), 3);
    }

View Full Code Here

        List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<Map<ColumnHandle, Object>>of(ImmutableMap.<ColumnHandle, Object>of(dsColumnHandle, "foo")));
        assertEquals(partitions.size(), 2);


        // foo=bar. Not a prunable column
        Expression nonMatching = new ComparisonExpression(Type.EQUAL, new QualifiedNameReference(new QualifiedName("foo")), new StringLiteral("bar"));
        DataSource dataSource = splitManager.getSplits(session, tableHandle, BooleanLiteral.TRUE_LITERAL, nonMatching, Predicates.<Partition>alwaysTrue(), symbols);
        List<Split> splits = ImmutableList.copyOf(dataSource.getSplits());
        // all splits found
        assertEquals(splits.size(), 4);
    }

View Full Code Here

        List<TaskSource> sources = new ArrayList<>();
        long sequenceId = 0;
        for (PlanNode sourceNode : subplan.getFragment().getSources()) {
            TableScanNode tableScan = (TableScanNode) sourceNode;


            DataSource dataSource = splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));


            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            for (Split split : dataSource.getSplits()) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
            }


            sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
        }

View Full Code Here

        List<Partition> partitions = getPartitions(session, handle, and(predicate, upstreamHint), partitionPredicate, mappings);


        ConnectorSplitManager connectorSplitManager = getConnectorSplitManager(handle);


        String connectorId = connectorSplitManager.getConnectorId();
        return new DataSource(connectorId, connectorSplitManager.getPartitionSplits(handle, partitions));
    }

View Full Code Here


        @Override
        public NodeSplits visitTableScan(TableScanNode node, Predicate<Partition> tableWriterPartitionPredicate)
        {
            // get dataSource for table
            DataSource dataSource = splitManager.getSplits(session,
                    node.getTable(),
                    node.getPartitionPredicate(),
                    node.getUpstreamPredicateHint(),
                    tableWriterPartitionPredicate,
                    node.getAssignments());

View Full Code Here

                    return node.getSource().accept(this, tableWriterPartitionPredicate);


                case SYSTEM: {
                    final double ratio = node.getSampleRatio();
                    NodeSplits nodeSplits = node.getSource().accept(this, tableWriterPartitionPredicate);
                    DataSource dataSource = nodeSplits.dataSource.get();
                    Iterable<Split> sampleIterable = Iterables.filter(dataSource.getSplits(), new Predicate<Split>()
                    {
                        public boolean apply(@Nullable Split input)
                        {
                            return ThreadLocalRandom.current().nextDouble() < ratio;
                        }
                    });
                    DataSource sampledDataSource = new DataSource(dataSource.getDataSourceName(), sampleIterable);


                    return new NodeSplits(node.getId(), sampledDataSource);
                }
                default:
                    throw new UnsupportedOperationException("Sampling is not supported for type " + node.getSampleType());

View Full Code Here

            TableWriter tableWriter = new TableWriter(node, shardManager);


            // get source splits
            NodeSplits nodeSplits = node.getSource().accept(this, tableWriter.getPartitionPredicate());
            checkState(nodeSplits.dataSource.isPresent(), "No splits present for import");
            DataSource dataSource = nodeSplits.dataSource.get();


            // record output
            outputReceivers.put(node.getId(), tableWriter.getOutputReceiver());


            // wrap splits with table writer info
            Iterable<Split> newSplits = tableWriter.wrapSplits(nodeSplits.planNodeId, dataSource.getSplits());
            return new NodeSplits(node.getId(), new DataSource(dataSource.getDataSourceName(), newSplits));
        }

View Full Code Here

            List<Partition> partitions = FluentIterable.from(getPartitions(node))
                    .filter(materializedViewPartitionPredicate)
                    .toList();


            // get dataSource for table
            DataSource dataSource = splitManager.getPartitionSplits(node.getTable(), partitions);


            return new NodeSplits(node.getId(), dataSource);
        }

View Full Code Here

                    return node.getSource().accept(this, materializedViewPartitionPredicate);


                case SYSTEM: {
                    NodeSplits nodeSplits = node.getSource().accept(this, materializedViewPartitionPredicate);
                    if (nodeSplits.dataSource.isPresent()) {
                        DataSource dataSource = nodeSplits.dataSource.get();
                        final double ratio = node.getSampleRatio();
                        Iterable<Split> sampleIterable = Iterables.filter(dataSource.getSplits(), new Predicate<Split>()
                        {
                            public boolean apply(@Nullable Split input)
                            {
                                return ThreadLocalRandom.current().nextDouble() < ratio;
                            }
                        });
                        DataSource sampledDataSource = new DataSource(dataSource.getDataSourceName(), sampleIterable);


                        return new NodeSplits(node.getId(), sampledDataSource);
                    }
                    else {
                        // table sampling on a sub query without splits is meaningless

View Full Code Here

0 1 2

TOP

Related Classes of com.facebook.presto.execution.DataSource

com.facebook.presto.split.SplitManager

com.facebook.presto.split.TestNativeSplitManager

com.facebook.presto.sql.planner.DistributedExecutionPlanner$Visitor

com.facebook.presto.util.LocalQueryRunner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.