/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.operator;
import com.facebook.presto.ExceededMemoryLimitException;
import com.facebook.presto.spi.ConnectorSession;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.util.concurrent.ListenableFuture;
import io.airlift.stats.CounterStat;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import javax.annotation.concurrent.ThreadSafe;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.concurrent.Executor;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static io.airlift.units.DataSize.Unit.BYTE;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@ThreadSafe
public class OperatorContext
{
private static final ThreadMXBean THREAD_MX_BEAN = ManagementFactory.getThreadMXBean();
private final int operatorId;
private final String operatorType;
private final DriverContext driverContext;
private final Executor executor;
private final AtomicLong intervalWallStart = new AtomicLong();
private final AtomicLong intervalCpuStart = new AtomicLong();
private final AtomicLong intervalUserStart = new AtomicLong();
private final AtomicLong addInputCalls = new AtomicLong();
private final AtomicLong addInputWallNanos = new AtomicLong();
private final AtomicLong addInputCpuNanos = new AtomicLong();
private final AtomicLong addInputUserNanos = new AtomicLong();
private final CounterStat inputDataSize = new CounterStat();
private final CounterStat inputPositions = new CounterStat();
private final AtomicLong getOutputCalls = new AtomicLong();
private final AtomicLong getOutputWallNanos = new AtomicLong();
private final AtomicLong getOutputCpuNanos = new AtomicLong();
private final AtomicLong getOutputUserNanos = new AtomicLong();
private final CounterStat outputDataSize = new CounterStat();
private final CounterStat outputPositions = new CounterStat();
private final AtomicLong blockedWallNanos = new AtomicLong();
private final AtomicLong finishCalls = new AtomicLong();
private final AtomicLong finishWallNanos = new AtomicLong();
private final AtomicLong finishCpuNanos = new AtomicLong();
private final AtomicLong finishUserNanos = new AtomicLong();
private final AtomicLong memoryReservation = new AtomicLong();
private final AtomicReference<Supplier<Object>> infoSupplier = new AtomicReference<>();
public OperatorContext(int operatorId, String operatorType, DriverContext driverContext, Executor executor)
{
checkArgument(operatorId >= 0, "operatorId is negative");
this.operatorId = operatorId;
this.operatorType = checkNotNull(operatorType, "operatorType is null");
this.driverContext = checkNotNull(driverContext, "driverContext is null");
this.executor = checkNotNull(executor, "executor is null");
}
public int getOperatorId()
{
return operatorId;
}
public String getOperatorType()
{
return operatorType;
}
public DriverContext getDriverContext()
{
return driverContext;
}
public ConnectorSession getSession()
{
return driverContext.getSession();
}
public boolean isDone()
{
return driverContext.isDone();
}
public void startIntervalTimer()
{
intervalWallStart.set(System.nanoTime());
intervalCpuStart.set(currentThreadCpuTime());
intervalUserStart.set(currentThreadUserTime());
}
public void recordAddInput(Page page)
{
addInputCalls.incrementAndGet();
addInputWallNanos.getAndAdd(nanosBetween(intervalWallStart.get(), System.nanoTime()));
addInputCpuNanos.getAndAdd(nanosBetween(intervalCpuStart.get(), currentThreadCpuTime()));
addInputUserNanos.getAndAdd(nanosBetween(intervalUserStart.get(), currentThreadUserTime()));
if (page != null) {
inputDataSize.update(page.getDataSize().toBytes());
inputPositions.update(page.getPositionCount());
}
}
public void recordGeneratedInput(DataSize dataSize, long positions, long readNanos)
{
inputDataSize.update(dataSize.toBytes());
inputPositions.update(positions);
addInputWallNanos.getAndAdd(readNanos);
}
public void recordGeneratedInput(DataSize dataSize, long positions)
{
recordGeneratedInput(dataSize, positions, 0);
}
public void recordGetOutput(Page page)
{
getOutputCalls.incrementAndGet();
getOutputWallNanos.getAndAdd(nanosBetween(intervalWallStart.get(), System.nanoTime()));
getOutputCpuNanos.getAndAdd(nanosBetween(intervalCpuStart.get(), currentThreadCpuTime()));
getOutputUserNanos.getAndAdd(nanosBetween(intervalUserStart.get(), currentThreadUserTime()));
if (page != null) {
outputDataSize.update(page.getDataSize().toBytes());
outputPositions.update(page.getPositionCount());
}
}
public void recordGeneratedOutput(DataSize dataSize, long positions)
{
outputDataSize.update(dataSize.toBytes());
outputPositions.update(positions);
}
public void recordBlocked(ListenableFuture<?> blocked)
{
checkNotNull(blocked, "blocked is null");
blocked.addListener(new Runnable()
{
private final long start = System.nanoTime();
@Override
public void run()
{
blockedWallNanos.getAndAdd(nanosBetween(start, System.nanoTime()));
}
}, executor);
}
public void recordFinish()
{
finishCalls.incrementAndGet();
finishWallNanos.getAndAdd(nanosBetween(intervalWallStart.get(), System.nanoTime()));
finishCpuNanos.getAndAdd(nanosBetween(intervalCpuStart.get(), currentThreadCpuTime()));
finishUserNanos.getAndAdd(nanosBetween(intervalUserStart.get(), currentThreadUserTime()));
}
public DataSize getMaxMemorySize()
{
return driverContext.getMaxMemorySize();
}
public DataSize getOperatorPreAllocatedMemory()
{
return driverContext.getOperatorPreAllocatedMemory();
}
public boolean reserveMemory(long bytes)
{
boolean result = driverContext.reserveMemory(bytes);
if (result) {
memoryReservation.getAndAdd(bytes);
}
return result;
}
public synchronized long setMemoryReservation(long newMemoryReservation)
{
checkArgument(newMemoryReservation >= 0, "newMemoryReservation is negative");
long delta = newMemoryReservation - memoryReservation.get();
// currently, operator memory is not be released
if (delta > 0 && !reserveMemory(delta)) {
throw new ExceededMemoryLimitException(getMaxMemorySize());
}
return newMemoryReservation;
}
public void setInfoSupplier(Supplier<Object> infoSupplier)
{
checkNotNull(infoSupplier, "infoProvider is null");
this.infoSupplier.set(infoSupplier);
}
public CounterStat getInputDataSize()
{
return inputDataSize;
}
public CounterStat getInputPositions()
{
return inputPositions;
}
public CounterStat getOutputDataSize()
{
return outputDataSize;
}
public CounterStat getOutputPositions()
{
return outputPositions;
}
public OperatorStats getOperatorStats()
{
Supplier<Object> infoSupplier = this.infoSupplier.get();
Object info = null;
if (infoSupplier != null) {
info = infoSupplier.get();
}
return new OperatorStats(
operatorId,
operatorType,
addInputCalls.get(),
new Duration(addInputWallNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(addInputCpuNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(addInputUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new DataSize(inputDataSize.getTotalCount(), BYTE).convertToMostSuccinctDataSize(),
inputPositions.getTotalCount(),
getOutputCalls.get(),
new Duration(getOutputWallNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(getOutputCpuNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(getOutputUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new DataSize(outputDataSize.getTotalCount(), BYTE).convertToMostSuccinctDataSize(),
outputPositions.getTotalCount(),
new Duration(blockedWallNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
finishCalls.get(),
new Duration(finishWallNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(finishCpuNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new Duration(finishUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(),
new DataSize(memoryReservation.get(), BYTE).convertToMostSuccinctDataSize(),
info);
}
private long currentThreadUserTime()
{
if (!driverContext.isCpuTimerEnabled()) {
return 0;
}
return THREAD_MX_BEAN.getCurrentThreadUserTime();
}
private long currentThreadCpuTime()
{
if (!driverContext.isCpuTimerEnabled()) {
return 0;
}
return THREAD_MX_BEAN.getCurrentThreadCpuTime();
}
private static long nanosBetween(long start, long end)
{
return Math.abs(end - start);
}
public static Function<OperatorContext, OperatorStats> operatorStatsGetter()
{
return new Function<OperatorContext, OperatorStats>()
{
public OperatorStats apply(OperatorContext operatorContext)
{
return operatorContext.getOperatorStats();
}
};
}
}