Package io.airlift.units

Examples of io.airlift.units.Duration


            throws Exception
    {
        return new DBI(new H2EmbeddedDataSource(new H2EmbeddedDataSourceConfig()
                .setFilename(new File(config.getDataDirectory(), "db/StorageManager").getAbsolutePath())
                .setMaxConnections(500)
                .setMaxConnectionWait(new Duration(1, SECONDS))));
    }
View Full Code Here


        int maxSleepMillis = 1;

        while (remainingNanos > 0) {
            // todo we need a much better way to determine if a task is unknown (e.g. not scheduled yet), done, or there is current no more data
            try {
                BufferResult result = taskManager.getTaskResults(taskId, outputId, pageSequenceId, DEFAULT_MAX_SIZE, new Duration(remainingNanos, NANOSECONDS));
                if (!result.isEmpty()) {
                    GenericEntity<?> entity = new GenericEntity<>(result.getElements(), new TypeToken<List<Page>>() {}.getType());
                    return Response.ok(entity).header(PRESTO_PAGE_SEQUENCE_ID, result.getStartingSequenceId()).build();
                }
                else if (result.isBufferClosed()) {
View Full Code Here

        @SuppressWarnings("resource")
        ExchangeClient exchangeClient = new ExchangeClient(createTestingBlockEncodingManager(),
                new DataSize(32, Unit.MEGABYTE),
                maxResponseSize,
                1,
                new Duration(1, TimeUnit.MINUTES),
                new TestingHttpClient(processor, executor),
                executor);

        exchangeClient.addLocation(location);
        exchangeClient.noMoreLocations();

        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
        assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
        assertEquals(exchangeClient.isClosed(), true);
        if (exchangeClient.getStatus().getBufferedPages() != 0) {
            assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
        }
        assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
View Full Code Here

        @SuppressWarnings("resource")
        ExchangeClient exchangeClient = new ExchangeClient(createTestingBlockEncodingManager(),
                new DataSize(32, Unit.MEGABYTE),
                maxResponseSize,
                1,
                new Duration(1, TimeUnit.MINUTES),
                new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))),
                executor);

        URI location1 = URI.create("http://localhost:8081/foo");
        processor.addPage(location1, createPage(1));
        processor.addPage(location1, createPage(2));
        processor.addPage(location1, createPage(3));
        processor.setComplete(location1);
        exchangeClient.addLocation(location1);

        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));

        assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
        assertEquals(exchangeClient.isClosed(), false);

        URI location2 = URI.create("http://localhost:8082/bar");
        processor.addPage(location2, createPage(4));
        processor.addPage(location2, createPage(5));
        processor.addPage(location2, createPage(6));
        processor.setComplete(location2);
        exchangeClient.addLocation(location2);

        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(4));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(5));
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(6));

        assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
        assertEquals(exchangeClient.isClosed(), false);

        exchangeClient.noMoreLocations();
        assertEquals(exchangeClient.isClosed(), true);
View Full Code Here

        @SuppressWarnings("resource")
        ExchangeClient exchangeClient = new ExchangeClient(createTestingBlockEncodingManager(),
                new DataSize(1, Unit.BYTE),
                maxResponseSize,
                1,
                new Duration(1, TimeUnit.MINUTES),
                new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))),
                executor);

        URI location = URI.create("http://localhost:8080");
        exchangeClient.addLocation(location);
        exchangeClient.noMoreLocations();
        assertEquals(exchangeClient.isClosed(), false);

        long start = System.nanoTime();

        // add a pages
        processor.addPage(location, createPage(1));
        processor.addPage(location, createPage(2));
        processor.addPage(location, createPage(3));
        processor.setComplete(location);

        // start fetching pages
        exchangeClient.scheduleRequestIfNecessary();
        // wait for a page to be fetched
        do {
            // there is no thread coordination here, so sleep is the best we can do
            assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
        while (exchangeClient.getStatus().getBufferedPages() == 0);

        // client should have sent a single request for a single page
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
        assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
        assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 1, 1, 1, "not scheduled");

        // remove the page and wait for the client to fetch another page
        assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(1));
        do {
            assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
        while (exchangeClient.getStatus().getBufferedPages() == 0);

        // client should have sent a single request for a single page
        assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 2, 2, 2, "not scheduled");
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
        assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);

        // remove the page and wait for the client to fetch another page
        assertPageEquals(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)), createPage(2));
        do {
            assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
        while (exchangeClient.getStatus().getBufferedPages() == 0);

        // client should have sent a single request for a single page
        assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 3, 3, 3, "not scheduled");
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
        assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);

        // remove last page
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));

        //  wait for client to decide there are no more pages
        assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
        assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
        assertEquals(exchangeClient.isClosed(), true);
        assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 4, 4, "not scheduled");
    }
View Full Code Here

        @SuppressWarnings("resource")
        ExchangeClient exchangeClient = new ExchangeClient(createTestingBlockEncodingManager(),
                new DataSize(1, Unit.BYTE),
                maxResponseSize, 1,
                new Duration(1, TimeUnit.MINUTES),
                new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))),
                executor);
        exchangeClient.addLocation(location);
        exchangeClient.noMoreLocations();

        // fetch a page
        assertEquals(exchangeClient.isClosed(), false);
        assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));

        // close client while pages are still available
        exchangeClient.close();
        assertEquals(exchangeClient.isClosed(), true);
        assertNull(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)));
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
        assertEquals(exchangeClient.getStatus().getBufferedBytes(), 0);

        // client should have sent only 2 requests: one to get all pages and once to get the done signal
        PageBufferClientStatus clientStatus = exchangeClient.getStatus().getPageBufferClientStatuses().get(0);
View Full Code Here

    public synchronized QueryInfo getQueryInfo(StageInfo rootStage)
    {
        QueryState state = queryState.get();

        Duration elapsedTime;
        if (endTime != null) {
            elapsedTime = new Duration(endTime.getMillis() - createTime.getMillis(), MILLISECONDS);
        }
        else {
            elapsedTime = Duration.nanosSince(createNanos);
        }

        // don't report failure info is query is marked as success
        FailureInfo failureInfo = null;
        ErrorCode errorCode = null;
        if (state != FINISHED) {
            failureInfo = toFailure(failureCause);
            errorCode = ErrorCodes.toErrorCode(failureCause);
        }

        int totalTasks = 0;
        int runningTasks = 0;
        int completedTasks = 0;

        int totalDrivers = 0;
        int queuedDrivers = 0;
        int runningDrivers = 0;
        int completedDrivers = 0;

        long totalMemoryReservation = 0;

        long totalScheduledTime = 0;
        long totalCpuTime = 0;
        long totalUserTime = 0;
        long totalBlockedTime = 0;

        long rawInputDataSize = 0;
        long rawInputPositions = 0;

        long processedInputDataSize = 0;
        long processedInputPositions = 0;

        long outputDataSize = 0;
        long outputPositions = 0;

        if (rootStage != null) {
            for (StageInfo stageInfo : getAllStages(rootStage)) {
                StageStats stageStats = stageInfo.getStageStats();
                totalTasks += stageStats.getTotalTasks();
                runningTasks += stageStats.getRunningTasks();
                completedTasks += stageStats.getCompletedTasks();

                totalDrivers += stageStats.getTotalDrivers();
                queuedDrivers += stageStats.getQueuedDrivers();
                runningDrivers += stageStats.getRunningDrivers();
                completedDrivers += stageStats.getCompletedDrivers();

                totalMemoryReservation += stageStats.getTotalMemoryReservation().toBytes();

                totalScheduledTime += stageStats.getTotalScheduledTime().roundTo(NANOSECONDS);
                totalCpuTime += stageStats.getTotalCpuTime().roundTo(NANOSECONDS);
                totalUserTime += stageStats.getTotalUserTime().roundTo(NANOSECONDS);
                totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(NANOSECONDS);

                if (stageInfo.getSubStages().isEmpty()) {
                    rawInputDataSize += stageStats.getRawInputDataSize().toBytes();
                    rawInputPositions += stageStats.getRawInputPositions();

                    processedInputDataSize += stageStats.getProcessedInputDataSize().toBytes();
                    processedInputPositions += stageStats.getProcessedInputPositions();
                }
            }

            StageStats outputStageStats = rootStage.getStageStats();
            outputDataSize += outputStageStats.getOutputDataSize().toBytes();
            outputPositions += outputStageStats.getOutputPositions();
        }

        QueryStats queryStats = new QueryStats(
                createTime,
                executionStartTime,
                lastHeartbeat,
                endTime,

                elapsedTime.convertToMostSuccinctTimeUnit(),
                queuedTime,
                analysisTime,
                distributedPlanningTime,
                totalPlanningTime,

                totalTasks,
                runningTasks,
                completedTasks,

                totalDrivers,
                queuedDrivers,
                runningDrivers,
                completedDrivers,

                new DataSize(totalMemoryReservation, BYTE).convertToMostSuccinctDataSize(),
                new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(),
                new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(),
                new Duration(totalUserTime, NANOSECONDS).convertToMostSuccinctTimeUnit(),
                new Duration(totalBlockedTime, NANOSECONDS).convertToMostSuccinctTimeUnit(),
                new DataSize(rawInputDataSize, BYTE).convertToMostSuccinctDataSize(),
                rawInputPositions,
                new DataSize(processedInputDataSize, BYTE).convertToMostSuccinctDataSize(),
                processedInputPositions,
                new DataSize(outputDataSize, BYTE).convertToMostSuccinctDataSize(),
View Full Code Here

    protected <T> ListenableFuture<T> createClient(Class<T> clientClass, int serverPort, final Duration connectDelay)
            throws TTransportException, InterruptedException, ExecutionException
    {
        HostAndPort address = HostAndPort.fromParts("localhost", serverPort);
        ThriftClientConfig config = new ThriftClientConfig().setConnectTimeout(new Duration(1, TimeUnit.SECONDS))
                                                            .setReadTimeout(new Duration(1, TimeUnit.SECONDS))
                                                            .setWriteTimeout(new Duration(1, TimeUnit.SECONDS));
        FramedClientConnector connector = new FramedClientConnector(address) {
            @Override
            public FramedClientChannel newThriftClientChannel(
                    Channel nettyChannel, Timer timer)
            {
View Full Code Here

    }

    protected <T> ListenableFuture<T> createHttpClient(Class<T> clientClass, int serverPort)
            throws TTransportException, InterruptedException, ExecutionException
    {
        ThriftClientConfig config = new ThriftClientConfig().setConnectTimeout(new Duration(1, TimeUnit.SECONDS))
                                                            .setReadTimeout(new Duration(1, TimeUnit.SECONDS))
                                                            .setWriteTimeout(new Duration(1, TimeUnit.SECONDS));
        HttpClientConnector connector =
                new HttpClientConnector(URI.create("http://localhost:" + serverPort + "/thrift/"));
        return new ThriftClient<>(clientManager, clientClass, config, "asyncTestClient").open(connector);
    }
View Full Code Here

        splitCompletionEvent(taskId, driverStats, cause.getClass().getName(), cause.getMessage());
    }

    private void splitCompletionEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage)
    {
        Duration timeToStart = null;
        if (driverStats.getStartTime() != null) {
            timeToStart = new Duration(driverStats.getStartTime().getMillis() - driverStats.getCreateTime().getMillis(), MILLISECONDS);
        }
        Duration timeToEnd = null;
        if (driverStats.getEndTime() != null) {
            timeToEnd = new Duration(driverStats.getEndTime().getMillis() - driverStats.getCreateTime().getMillis(), MILLISECONDS);
        }

        try {
            eventClient.post(
                    new SplitCompletionEvent(
View Full Code Here

TOP

Related Classes of io.airlift.units.Duration

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.