Package com.facebook.presto.spi

Examples of com.facebook.presto.spi.SplitSource

112
113
114
115
116
117
118
119
120
            List<Partition> partitions = FluentIterable.from(getPartitions(node))
                    .filter(materializedViewPartitionPredicate)
                    .toList();

            // get dataSource for table
            SplitSource splitSource = splitManager.getPartitionSplits(node.getTable(), partitions);

            return new NodeSplits(node.getId(), splitSource);
        }
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
                    return node.getSource().accept(this, materializedViewPartitionPredicate);

                case SYSTEM: {
                    NodeSplits nodeSplits = node.getSource().accept(this, materializedViewPartitionPredicate);
                    if (nodeSplits.getDataSource().isPresent()) {
                        SplitSource sampledSplitSource = new SampledSplitSource(nodeSplits.getDataSource().get(), node.getSampleRatio());
                        return new NodeSplits(node.getId(), sampledSplitSource);
                    }
                    else {
                        // table sampling on a sub query without splits is meaningless
                        return nodeSplits;
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
            MaterializedViewWriter materializedViewWriter = new MaterializedViewWriter(node, shardManager);

            // get source splits
            NodeSplits nodeSplits = node.getSource().accept(this, materializedViewWriter.getPartitionPredicate());
            checkState(nodeSplits.getDataSource().isPresent(), "No splits present for import");
            SplitSource splitSource = nodeSplits.getDataSource().get();

            // record output
            outputReceivers.put(node.getId(), materializedViewWriter.getOutputReceiver());

            // wrap splits with table writer info
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
        ImmutableList.Builder<Split> splits = ImmutableList.builder();
        for (int i = 0; i < splitCount; i++) {
            splits.add(new DualSplit(HostAddress.fromString("127.0.0.1")));
        }
        SplitSource splitSource = new FixedSplitSource(null, splits.build());

        return new StageExecutionPlan(testFragment,
                Optional.of(splitSource),
                ImmutableList.<StageExecutionPlan>of()
        );
View Full Code Here
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
            throws InterruptedException
    {
        AtomicInteger nextTaskId = new AtomicInteger(0);
        long getSplitStart = System.nanoTime();

        SplitSource splitSource = this.dataSource.get();
        while (!splitSource.isFinished()) {
            getSplitDistribution.add(System.nanoTime() - getSplitStart);

            // if query has been canceled, exit cleanly; query will never run regardless
            if (getState().isDone()) {
                break;
            }

            Multimap<Node, Split> nodeSplits = ArrayListMultimap.create();
            for (Split split : splitSource.getNextBatch(splitBatchSize)) {
                Node node = chooseNode(nodeSelector, split, nextTaskId);
                nodeSplits.put(node, split);
            }

            for (Entry<Node, Collection<Split>> taskSplits : nodeSplits.asMap().entrySet()) {
View Full Code Here
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
            throws InterruptedException
    {
        AtomicInteger nextTaskId = new AtomicInteger(0);
        long getSplitStart = System.nanoTime();

        SplitSource splitSource = this.dataSource.get();
        while (!splitSource.isFinished()) {
            getSplitDistribution.add(System.nanoTime() - getSplitStart);

            // if query has been canceled, exit cleanly; query will never run regardless
            if (getState().isDone()) {
                break;
            }

            Set<Split> pendingSplits = ImmutableSet.copyOf(splitSource.getNextBatch(splitBatchSize));
            while (!pendingSplits.isEmpty() && !getState().isDone()) {
                Multimap<Node, Split> splitAssignment = nodeSelector.computeAssignments(pendingSplits);
                pendingSplits = ImmutableSet.copyOf(Sets.difference(pendingSplits, ImmutableSet.copyOf(splitAssignment.values())));

                assignSplits(nextTaskId, splitAssignment);
View Full Code Here
110
111
112
113
114
115
116
117
118
            List<Partition> partitions = FluentIterable.from(getPartitions(node))
                    .filter(materializedViewPartitionPredicate)
                    .toList();

            // get dataSource for table
            SplitSource splitSource = splitManager.getPartitionSplits(node.getTable(), partitions);

            return new NodeSplits(node.getId(), splitSource);
        }
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
                    return node.getSource().accept(this, materializedViewPartitionPredicate);

                case SYSTEM: {
                    NodeSplits nodeSplits = node.getSource().accept(this, materializedViewPartitionPredicate);
                    if (nodeSplits.getDataSource().isPresent()) {
                        SplitSource sampledSplitSource = new SampledSplitSource(nodeSplits.getDataSource().get(), node.getSampleRatio());
                        return new NodeSplits(node.getId(), sampledSplitSource);
                    }
                    else {
                        // table sampling on a sub query without splits is meaningless
                        return nodeSplits;
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
            MaterializedViewWriter materializedViewWriter = new MaterializedViewWriter(node, shardManager);

            // get source splits
            NodeSplits nodeSplits = node.getSource().accept(this, materializedViewWriter.getPartitionPredicate());
            checkState(nodeSplits.getDataSource().isPresent(), "No splits present for import");
            SplitSource splitSource = nodeSplits.getDataSource().get();

            // record output
            outputReceivers.put(node.getId(), materializedViewWriter.getOutputReceiver());

            // wrap splits with table writer info
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(client.getColumnHandles(table).values());
        Map<String, Integer> columnIndex = indexColumns(columnHandles);

        PartitionResult partitionResult = client.getPartitions(table, TupleDomain.all());
        assertEquals(partitionResult.getPartitions().size(), 1);
        SplitSource splitSource = client.getPartitionSplits(table, partitionResult.getPartitions());

        long sum = 0;
        for (Split split : getAllSplits(splitSource)) {
            try (RecordCursor cursor = client.getRecordSet(split, columnHandles).cursor()) {
                while (cursor.advanceNextPosition()) {
View Full Code Here
TOP

Related Classes of com.facebook.presto.spi.SplitSource

  • com.facebook.presto.execution.SqlStageExecution
  • com.facebook.presto.execution.TestSqlStageExecution
  • com.facebook.presto.execution.TestSqlTaskExecution
  • com.facebook.presto.execution.TestSqlTaskManager
  • com.facebook.presto.hive.AbstractTestHiveClient
  • com.facebook.presto.hive.AbstractTestHiveClientS3
  • com.facebook.presto.split.TestNativeSplitManager
  • com.facebook.presto.sql.planner.DistributedExecutionPlanner$Visitor
  • com.facebook.presto.util.LocalQueryRunner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.