Package com.facebook.presto.spi

Examples of com.facebook.presto.spi.SplitSource

128
129
130
131
132
133
134
135
136
137
138
139
        List<Partition> partitions = partitionResult.getPartitions();
        TupleDomain columnUnionedTupleDomain = partitions.get(0).getTupleDomain().columnWiseUnion(partitions.get(1).getTupleDomain());
        assertEquals(columnUnionedTupleDomain, TupleDomain.withColumnDomains(ImmutableMap.of(dsColumnHandle, Domain.create(SortedRangeSet.of(Range.equal("1"), Range.equal("2")), false))));

        SplitSource splitSource = nativeSplitManager.getPartitionSplits(tableHandle, partitions);
        int splitCount = 0;
        while (!splitSource.isFinished()) {
            splitCount += splitSource.getNextBatch(1000).size();
        }
        assertEquals(splitCount, 4);
    }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
        List<TaskSource> sources = new ArrayList<>();
        long sequenceId = 0;
        for (PlanNode sourceNode : subplan.getFragment().getSources()) {
            TableScanNode tableScan = (TableScanNode) sourceNode;

            SplitSource splitSource = splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));

            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            while (!splitSource.isFinished()) {
                try {
                    for (Split split : splitSource.getNextBatch(1000)) {
                        scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
                    }
                }
                catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
View Full Code Here
418
419
420
421
422
423
424
425
426
427
428
429
430
    private Split getLocalQuerySplit(TableHandle tableHandle)
    {
        try {
            List<Partition> partitions = splitManager.getPartitions(tableHandle, Optional.<TupleDomain>absent()).getPartitions();
            SplitSource splitSource = splitManager.getPartitionSplits(tableHandle, partitions);
            Split split = Iterables.getOnlyElement(splitSource.getNextBatch(1000));
            checkState(splitSource.isFinished(), "Expected only one split for a local query");
            return split;
        }
        catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw Throwables.propagate(e);
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
        List<TaskSource> sources = new ArrayList<>();
        long sequenceId = 0;
        for (PlanNode sourceNode : subplan.getFragment().getSources()) {
            TableScanNode tableScan = (TableScanNode) sourceNode;

            SplitSource splitSource = splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));

            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            while (!splitSource.isFinished()) {
                try {
                    for (Split split : splitSource.getNextBatch(1000)) {
                        scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
                    }
                }
                catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
                ImmutableMap.<Symbol, Type>of(symbol, Type.VARCHAR),
                PlanDistribution.SOURCE,
                tableScanNodeId,
                OutputPartitioning.NONE,
                ImmutableList.<Symbol>of());
        SplitSource splitSource = new FixedSplitSource(null, ImmutableList.copyOf(Collections.nCopies(splitCount, split)));

        return new StageExecutionPlan(testFragment,
                Optional.<SplitSource>of(splitSource),
                ImmutableList.<StageExecutionPlan>of(),
                ImmutableMap.<PlanNodeId, OutputReceiver>of());
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
        DualSplitManager dualSplitManager = new DualSplitManager(new InMemoryNodeManager());
        PartitionResult partitionResult = dualSplitManager.getPartitions(tableHandle, TupleDomain.all());

        split = null;
        SplitSource splitSource = dualSplitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());
        while (!splitSource.isFinished()) {
            List<Split> nextBatch = splitSource.getNextBatch(1000);
            if (!nextBatch.isEmpty()) {
                assertNull(split);
                split = Iterables.getOnlyElement(nextBatch);
            }
        }
View Full Code Here
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
            throws InterruptedException
    {
        AtomicInteger nextTaskId = new AtomicInteger(0);
        long getSplitStart = System.nanoTime();

        SplitSource splitSource = this.dataSource.get();
        while (!splitSource.isFinished()) {
            getSplitDistribution.add(System.nanoTime() - getSplitStart);

            // if query has been canceled, exit cleanly; query will never run regardless
            if (getState().isDone()) {
                break;
            }

            Set<Split> pendingSplits = ImmutableSet.copyOf(splitSource.getNextBatch(splitBatchSize));
            while (!pendingSplits.isEmpty() && !getState().isDone()) {
                Multimap<Node, Split> splitAssignment = nodeSelector.computeAssignments(pendingSplits);
                pendingSplits = ImmutableSet.copyOf(Sets.difference(pendingSplits, ImmutableSet.copyOf(splitAssignment.values())));

                assignSplits(nextTaskId, splitAssignment);
View Full Code Here
85
86
87
88
89
90
91
92
93
    {
        @Override
        public Optional<SplitSource> visitTableScan(TableScanNode node, Void context)
        {
            // get dataSource for table
            SplitSource splitSource = splitManager.getPartitionSplits(node.getTable(), getPartitions(node));

            return Optional.of(splitSource);
        }
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
                    return node.getSource().accept(this, context);

                case SYSTEM:
                    Optional<SplitSource> nodeSplits = node.getSource().accept(this, context);
                    if (nodeSplits.isPresent()) {
                        SplitSource sampledSplitSource = new SampledSplitSource(nodeSplits.get(), node.getSampleRatio());
                        return Optional.of(sampledSplitSource);
                    }
                    // table sampling on a sub query without splits is meaningless
                    return nodeSplits;
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
                continue;
            }

            TableScanNode tableScan = (TableScanNode) sourceNode;

            SplitSource splitSource = splitManager.getPartitionSplits(tableScan.getTable(), getPartitions(tableScan));

            ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
            while (!splitSource.isFinished()) {
                try {
                    for (Split split : splitSource.getNextBatch(1000)) {
                        scheduledSplits.add(new ScheduledSplit(sequenceId++, split));
                    }
                }
                catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
View Full Code Here
TOP

Related Classes of com.facebook.presto.spi.SplitSource

  • com.facebook.presto.execution.SqlStageExecution
  • com.facebook.presto.execution.TestSqlStageExecution
  • com.facebook.presto.execution.TestSqlTaskExecution
  • com.facebook.presto.execution.TestSqlTaskManager
  • com.facebook.presto.hive.AbstractTestHiveClient
  • com.facebook.presto.hive.AbstractTestHiveClientS3
  • com.facebook.presto.split.TestNativeSplitManager
  • com.facebook.presto.sql.planner.DistributedExecutionPlanner$Visitor
  • com.facebook.presto.util.LocalQueryRunner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.