Examples of OutputBuffers


Examples of com.facebook.presto.OutputBuffers

        int splitCount = 20;
        ExecutorService remoteTaskExecutor = Executors.newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor"));
        MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor);
        ExecutorService executor = Executors.newCachedThreadPool(daemonThreadsNamed("stageExecutor"));

        OutputBuffers outputBuffers = INITIAL_EMPTY_OUTPUT_BUFFERS
                .withBuffer("out", new UnpartitionedPagePartitionFunction())
                .withNoMoreBufferIds();

        StageExecutionPlan tableScanPlan = createTableScanPlan("test", splitCount);
        return new SqlStageExecution(new QueryId("query"),
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

            StageExecutionPlan joinPlan = createJoinPlan("A");

            InMemoryNodeManager nodeManager = new InMemoryNodeManager();
            nodeManager.addNode("foo", new PrestoNode("other", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN));

            OutputBuffers outputBuffers = INITIAL_EMPTY_OUTPUT_BUFFERS
                    .withBuffer("out", new UnpartitionedPagePartitionFunction())
                    .withNoMoreBufferIds();

            stageExecution = new SqlStageExecution(new QueryId("query"),
                    new MockLocationFactory(),
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

    public synchronized void parentNodesAdded(List<Node> parentNodes, boolean noMoreParentNodes)
    {
        checkNotNull(parentNodes, "parentNodes is null");

        // get the current buffers
        OutputBuffers startingOutputBuffers = nextOutputBuffers != null ? nextOutputBuffers : currentOutputBuffers;

        // add new buffers
        OutputBuffers newOutputBuffers;
        if (fragment.getOutputPartitioning() == OutputPartitioning.NONE) {
            ImmutableMap.Builder<String, PagePartitionFunction> newBuffers = ImmutableMap.builder();
            for (Node parentNode : parentNodes) {
                newBuffers.put(parentNode.getNodeIdentifier(), new UnpartitionedPagePartitionFunction());
            }
            newOutputBuffers = startingOutputBuffers.withBuffers(newBuffers.build());

            // no more flag
            if (noMoreParentNodes) {
                newOutputBuffers = newOutputBuffers.withNoMoreBufferIds();
            }
        }
        else if (fragment.getOutputPartitioning() == OutputPartitioning.HASH) {
            checkArgument(noMoreParentNodes, "Hash partitioned output requires all parent nodes be added in a single call");

            ImmutableMap.Builder<String, PagePartitionFunction> buffers = ImmutableMap.builder();
            for (int nodeIndex = 0; nodeIndex < parentNodes.size(); nodeIndex++) {
                Node node = parentNodes.get(nodeIndex);
                buffers.put(node.getNodeIdentifier(), new HashPagePartitionFunction(nodeIndex, parentNodes.size(), fragment.getPartitioningChannels()));
            }

            newOutputBuffers = startingOutputBuffers
                    .withBuffers(buffers.build())
                    .withNoMoreBufferIds();
        }
        else {
            throw new UnsupportedOperationException("Unsupported output partitioning " + fragment.getOutputPartitioning());
        }

        // only notify scheduler and tasks if the buffers changed
        if (newOutputBuffers.getVersion() != startingOutputBuffers.getVersion()) {
            this.nextOutputBuffers = newOutputBuffers;
            this.notifyAll();
        }
    }
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

                .putAll(exchangeLocations.get())
                .putAll(newExchangeLocations)
                .build());

        // get new output buffer and update output buffer state
        OutputBuffers outputBuffers = updateToNextOutputBuffers();

        // finished state must be decided before update to avoid race conditions
        boolean finished = allSourceComplete && outputBuffers.isNoMoreBufferIds();

        // update tasks
        for (RemoteTask task : tasks.values()) {
            for (Entry<PlanNodeId, URI> entry : newExchangeLocations.entries()) {
                Split remoteSplit = createRemoteSplitFor(task.getNodeId(), entry.getValue());
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

    {
        TaskInfo taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(),
                new OutputBuffers(ImmutableSet.<String>of(), false));
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.getTaskInfo(taskInfo.getTaskId(), false);
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(new TaskSource(tableScanNodeId, ImmutableSet.<ScheduledSplit>of(), true)),
                new OutputBuffers(ImmutableSet.<String>of(), true));
        assertEquals(taskInfo.getState(), TaskState.FINISHED);

        taskInfo = sqlTaskManager.getTaskInfo(taskInfo.getTaskId(), false);
        assertEquals(taskInfo.getState(), TaskState.FINISHED);
    }
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

    {
        TaskInfo taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(new TaskSource(tableScanNodeId, ImmutableSet.of(new ScheduledSplit(0, split)), true)),
                new OutputBuffers(ImmutableSet.of("out"), true));
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.getTaskInfo(taskInfo.getTaskId(), false);
        assertEquals(taskInfo.getState(), TaskState.RUNNING);
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

    {
        TaskInfo taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(),
                new OutputBuffers(ImmutableSet.<String>of(), false));
        assertEquals(taskInfo.getState(), TaskState.RUNNING);
        assertNull(taskInfo.getStats().getEndTime());

        taskInfo = sqlTaskManager.getTaskInfo(taskInfo.getTaskId(), false);
        assertEquals(taskInfo.getState(), TaskState.RUNNING);
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

    {
        TaskInfo taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(new TaskSource(tableScanNodeId, ImmutableSet.of(new ScheduledSplit(0, split)), true)),
                new OutputBuffers(ImmutableSet.of("out"), true));
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.getTaskInfo(taskInfo.getTaskId(), false);
        assertEquals(taskInfo.getState(), TaskState.RUNNING);
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

        TaskInfo taskInfo = sqlTaskManager.updateTask(session,
                taskId,
                testFragment,
                ImmutableList.<TaskSource>of(),
                new OutputBuffers(ImmutableSet.<String>of(), false));
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.cancelTask(taskId);
        assertEquals(taskInfo.getState(), TaskState.CANCELED);
View Full Code Here

Examples of com.facebook.presto.OutputBuffers

        // add three items
        for (int i = 0; i < 3; i++) {
            addPage(sharedBuffer, createPage(i));
        }

        OutputBuffers outputBuffers = INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer("first", new UnpartitionedPagePartitionFunction());

        // add a queue
        sharedBuffer.setOutputBuffers(outputBuffers);
        assertQueueState(sharedBuffer, "first", 3, 0);

        // get the three elements
        assertBufferResultEquals(sharedBuffer.get("first", 0, sizeOfPages(10), NO_WAIT), bufferResult(0, createPage(0), createPage(1), createPage(2)));
        // pages not acknowledged yet so state is the same
        assertQueueState(sharedBuffer, "first", 3, 0);

        // try to get some more pages (acknowledge first three pages)
        assertBufferResultEquals(sharedBuffer.get("first", 3, sizeOfPages(10), NO_WAIT), emptyResults(3, false));
        // pages now acknowledged
        assertQueueState(sharedBuffer, "first", 0, 3);

        // fill the buffer (we already added 3 pages)
        for (int i = 3; i < 10; i++) {
            addPage(sharedBuffer, createPage(i));
        }
        assertQueueState(sharedBuffer, "first", 7, 3);

        // try to add one more page, which should block
        ListenableFuture<?> future = enqueuePage(sharedBuffer, createPage(10));

        // remove a page
        assertBufferResultEquals(sharedBuffer.get("first", 3, sizeOfPages(1), NO_WAIT), bufferResult(3, createPage(3)));
        // page not acknowledged yet so state is the same
        assertQueueState(sharedBuffer, "first", 7, 3);

        // we should still be blocked
        assertFalse(future.isDone());

        //
        // add another buffer and verify it sees all pages
        outputBuffers = outputBuffers.withBuffer("second", new UnpartitionedPagePartitionFunction());
        sharedBuffer.setOutputBuffers(outputBuffers);
        assertQueueState(sharedBuffer, "second", 10, 0);
        assertBufferResultEquals(sharedBuffer.get("second", 0, sizeOfPages(10), NO_WAIT), bufferResult(0, createPage(0),
                createPage(1),
                createPage(2),
                createPage(3),
                createPage(4),
                createPage(5),
                createPage(6),
                createPage(7),
                createPage(8),
                createPage(9)));
        // page not acknowledged yet so state is the same
        assertQueueState(sharedBuffer, "second", 10, 0);
        // acknowledge the 10 pages
        assertBufferResultEquals(sharedBuffer.get("second", 10, sizeOfPages(10), NO_WAIT), emptyResults(10, false));
        assertQueueState(sharedBuffer, "second", 0, 10);

        //
        // tell shared buffer there will be no more queues
        outputBuffers = outputBuffers.withNoMoreBufferIds();
        sharedBuffer.setOutputBuffers(outputBuffers);

        // since both queues consumed the first three pages, the blocked page future from above should be done
        future.get(1, TimeUnit.SECONDS);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.