/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.runtime.io.channels;
import eu.stratosphere.core.io.IOReadableWritable;
import eu.stratosphere.nephele.event.task.AbstractEvent;
import eu.stratosphere.nephele.event.task.AbstractTaskEvent;
import eu.stratosphere.nephele.jobgraph.JobID;
import eu.stratosphere.runtime.io.Buffer;
import eu.stratosphere.runtime.io.gates.InputChannelResult;
import eu.stratosphere.runtime.io.network.bufferprovider.BufferAvailabilityListener;
import eu.stratosphere.runtime.io.network.bufferprovider.BufferProvider;
import eu.stratosphere.runtime.io.network.Envelope;
import eu.stratosphere.runtime.io.gates.InputGate;
import eu.stratosphere.runtime.io.serialization.AdaptiveSpanningRecordDeserializer;
import eu.stratosphere.runtime.io.serialization.RecordDeserializer;
import eu.stratosphere.runtime.io.serialization.RecordDeserializer.DeserializationResult;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Queue;
/**
* InputChannel is an abstract base class to all different kinds of concrete
* input channels that can be used. Input channels are always parameterized to
* a specific type that can be transported through the channel.
* @param <T> The Type of the record that can be transported through the channel.
*/
public class InputChannel<T extends IOReadableWritable> extends Channel implements BufferProvider {
private final InputGate<T> inputGate;
/**
* The log object used to report warnings and errors.
*/
private static final Log LOG = LogFactory.getLog(InputChannel.class);
/**
* The deserializer used to deserialize records.
*/
private final RecordDeserializer<T> deserializer;
/**
* Buffer for the uncompressed (raw) data.
*/
private Buffer dataBuffer;
private AbstractTaskEvent currentEvent;
/**
* The exception observed in this channel while processing the buffers. Checked and thrown
* per-buffer.
*/
private volatile IOException ioException;
/**
* Stores the number of bytes read through this input channel since its instantiation.
*/
private long amountOfDataTransmitted;
private volatile boolean brokerAggreedToCloseChannel;
// -------------------------------------------------------------------------------------------
private int lastReceivedEnvelope = -1;
private ChannelID lastSourceID = null;
private boolean destroyCalled = false;
// ----------------------
private Queue<Envelope> queuedEnvelopes = new ArrayDeque<Envelope>();
private Iterator<AbstractEvent> pendingEvents;
/**
* Constructs an input channel with a given input gate associated.
*
* @param inputGate
* the input gate this channel is connected to
* @param channelIndex
* the index of the channel in the input gate
* @param channelID
* the ID of the channel
* @param connectedChannelID
* the ID of the channel this channel is connected to
*/
public InputChannel(final InputGate<T> inputGate, final int channelIndex, final ChannelID channelID,
final ChannelID connectedChannelID, ChannelType type) {
super(channelIndex, channelID, connectedChannelID, type);
this.inputGate = inputGate;
this.deserializer = new AdaptiveSpanningRecordDeserializer<T>();
}
/**
* Returns the input gate associated with the input channel.
*
* @return the input gate associated with the input channel.
*/
public InputGate<T> getInputGate() {
return this.inputGate;
}
/**
* Reads a record from the input channel. If currently no record is available the method
* returns <code>null</code>. If the channel is closed (i.e. no more records will be received), the method
* throws an {@link EOFException}.
*
* @return a record that has been transported through the channel or <code>null</code> if currently no record is
* available
* @throws IOException
* thrown if the input channel is already closed {@link EOFException} or a transmission error has occurred
*/
// public abstract InputChannelResult readRecord(T target) throws IOException;
/**
* Immediately closes the input channel. The corresponding output channels are
* notified if necessary. Any remaining records in any buffers or queue is considered
* irrelevant and is discarded.
*
* @throws InterruptedException
* thrown if the thread is interrupted while waiting for the channel to close
* @throws IOException
* thrown if an I/O error occurs while closing the channel
*/
// public abstract void close() throws IOException, InterruptedException;
@Override
public boolean isInputChannel() {
return true;
}
@Override
public JobID getJobID() {
return this.inputGate.getJobID();
}
// public abstract AbstractTaskEvent getCurrentEvent();
private DeserializationResult lastDeserializationResult;
public InputChannelResult readRecord(T target) throws IOException {
if (this.dataBuffer == null) {
if (isClosed()) {
return InputChannelResult.END_OF_STREAM;
}
// get the next element we need to handle (buffer or event)
BufferOrEvent boe = getNextBufferOrEvent();
if (boe == null) {
throw new IllegalStateException("Input channel was queries for data even though none was announced available.");
}
// handle events
if (boe.isEvent())
{
// sanity check: an event may only come after a complete record.
if (this.deserializer.hasUnfinishedData()) {
throw new IllegalStateException("Channel received an event before completing the current partial record.");
}
AbstractEvent evt = boe.getEvent();
if (evt.getClass() == ChannelCloseEvent.class) {
this.brokerAggreedToCloseChannel = true;
return InputChannelResult.END_OF_STREAM;
}
else if (evt.getClass() == EndOfSuperstepEvent.class) {
return InputChannelResult.END_OF_SUPERSTEP;
}
else if (evt instanceof AbstractTaskEvent) {
this.currentEvent = (AbstractTaskEvent) evt;
return InputChannelResult.TASK_EVENT;
}
else {
LOG.error("Received unknown event: " + evt);
return InputChannelResult.NONE;
}
} else {
// buffer case
this.dataBuffer = boe.getBuffer();
this.deserializer.setNextMemorySegment(this.dataBuffer.getMemorySegment(), this.dataBuffer.size());
}
}
DeserializationResult deserializationResult = this.deserializer.getNextRecord(target);
this.lastDeserializationResult = deserializationResult;
if (deserializationResult.isBufferConsumed()) {
releasedConsumedReadBuffer(this.dataBuffer);
this.dataBuffer = null;
}
if (deserializationResult == DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER) {
return InputChannelResult.INTERMEDIATE_RECORD_FROM_BUFFER;
} else if (deserializationResult == DeserializationResult.LAST_RECORD_FROM_BUFFER) {
return InputChannelResult.LAST_RECORD_FROM_BUFFER;
} else if (deserializationResult == DeserializationResult.PARTIAL_RECORD) {
return InputChannelResult.NONE;
} else {
throw new IllegalStateException();
}
}
@Override
public ChannelType getChannelType() {
return null;
}
@Override
public boolean isClosed() throws IOException{
if (this.ioException != null) {
throw new IOException("An error occurred in the channel: " + this.ioException.getMessage(), this.ioException);
} else {
return this.brokerAggreedToCloseChannel;
}
}
public void close() throws IOException, InterruptedException {
this.deserializer.clear();
if (this.dataBuffer != null) {
releasedConsumedReadBuffer(this.dataBuffer);
this.dataBuffer = null;
}
// This code fragment makes sure the isClosed method works in case the channel input has not been fully consumed
while (!this.brokerAggreedToCloseChannel)
{
BufferOrEvent next = getNextBufferOrEvent();
if (next != null) {
if (next.isEvent()) {
if (next.getEvent() instanceof ChannelCloseEvent) {
this.brokerAggreedToCloseChannel = true;
}
} else {
releasedConsumedReadBuffer(next.getBuffer());
}
} else {
Thread.sleep(200);
}
}
// Send close event to indicate the input channel has successfully
// processed all data it is interested in.
transferEventToOutputChannel(new ChannelCloseEvent());
}
private void releasedConsumedReadBuffer(Buffer buffer) {
this.amountOfDataTransmitted += buffer.size();
buffer.recycleBuffer();
}
public void notifyGateThatInputIsAvailable() {
this.getInputGate().notifyRecordIsAvailable(getIndex());
}
@Override
public void transferEvent(AbstractEvent event) throws IOException, InterruptedException {
transferEventToOutputChannel(event);
}
public void reportIOException(IOException ioe) {
this.ioException = ioe;
}
@Override
public void releaseAllResources() {
this.brokerAggreedToCloseChannel = true;
this.deserializer.clear();
// The buffers are recycled by the input channel wrapper
}
/**
* Notify the channel that a data unit has been consumed.
*/
public void notifyDataUnitConsumed() {
this.getInputGate().notifyDataUnitConsumed(getIndex());
}
public AbstractTaskEvent getCurrentEvent() {
AbstractTaskEvent e = this.currentEvent;
this.currentEvent = null;
return e;
}
// InputChannelContext
@Override
public void queueEnvelope(Envelope envelope) {
// The sequence number of the envelope to be queued
final int sequenceNumber = envelope.getSequenceNumber();
synchronized (this.queuedEnvelopes) {
if (this.destroyCalled) {
final Buffer buffer = envelope.getBuffer();
if (buffer != null) {
buffer.recycleBuffer();
}
return;
}
final int expectedSequenceNumber = this.lastReceivedEnvelope + 1;
if (sequenceNumber != expectedSequenceNumber) {
// This is a problem, now we are actually missing some data
reportIOException(new IOException("Expected data packet " + expectedSequenceNumber + " but received " + sequenceNumber));
// notify that something (an exception) is available
notifyGateThatInputIsAvailable();
if (LOG.isDebugEnabled()) {
LOG.debug("Input channel " + this.toString() + " expected envelope " + expectedSequenceNumber
+ " but received " + sequenceNumber);
}
// rescue the buffer
final Buffer buffer = envelope.getBuffer();
if (buffer != null) {
buffer.recycleBuffer();
}
} else {
this.queuedEnvelopes.add(envelope);
this.lastReceivedEnvelope = sequenceNumber;
this.lastSourceID = envelope.getSource();
// Notify the channel about the new data. notify as much as there is (buffer plus once per event)
if (envelope.getBuffer() != null) {
notifyGateThatInputIsAvailable();
}
List<? extends AbstractEvent> events = envelope.deserializeEvents();
if (events != null) {
for (int i = 0; i < events.size(); i++) {
notifyGateThatInputIsAvailable();
}
}
}
}
}
@Override
public void destroy() {
final Queue<Buffer> buffersToRecycle = new ArrayDeque<Buffer>();
synchronized (this.queuedEnvelopes) {
this.destroyCalled = true;
while (!this.queuedEnvelopes.isEmpty()) {
final Envelope envelope = this.queuedEnvelopes.poll();
if (envelope.getBuffer() != null) {
buffersToRecycle.add(envelope.getBuffer());
}
}
}
while (!buffersToRecycle.isEmpty()) {
buffersToRecycle.poll().recycleBuffer();
}
}
public void logQueuedEnvelopes() {
int numberOfQueuedEnvelopes = 0;
int numberOfQueuedEnvelopesWithMemoryBuffers = 0;
int numberOfQueuedEnvelopesWithFileBuffers = 0;
synchronized (this.queuedEnvelopes) {
final Iterator<Envelope> it = this.queuedEnvelopes.iterator();
while (it.hasNext()) {
final Envelope envelope = it.next();
++numberOfQueuedEnvelopes;
final Buffer buffer = envelope.getBuffer();
if (buffer == null) {
continue;
}
++numberOfQueuedEnvelopesWithMemoryBuffers;
}
}
System.out.println("\t\t" + this.toString() + ": " + numberOfQueuedEnvelopes + " ("
+ numberOfQueuedEnvelopesWithMemoryBuffers + ", " + numberOfQueuedEnvelopesWithFileBuffers + ")");
}
@Override
public Buffer requestBuffer(int minBufferSize) throws IOException {
return this.inputGate.requestBuffer(minBufferSize);
}
@Override
public Buffer requestBufferBlocking(int minBufferSize) throws IOException, InterruptedException {
return this.inputGate.requestBufferBlocking(minBufferSize);
}
@Override
public int getBufferSize() {
return this.inputGate.getBufferSize();
}
@Override
public void reportAsynchronousEvent() {
this.inputGate.reportAsynchronousEvent();
}
@Override
public BufferAvailabilityRegistration registerBufferAvailabilityListener(BufferAvailabilityListener listener) {
return this.inputGate.registerBufferAvailabilityListener(listener);
}
// ChannelBroker
public BufferOrEvent getNextBufferOrEvent() throws IOException {
// return pending events first
if (this.pendingEvents != null) {
// if the field is not null, it must always have a next value!
BufferOrEvent next = new BufferOrEvent(this.pendingEvents.next());
if (!this.pendingEvents.hasNext()) {
this.pendingEvents = null;
}
return next;
}
// if no events are pending, get the next buffer
Envelope nextEnvelope;
synchronized (this.queuedEnvelopes) {
if (this.queuedEnvelopes.isEmpty()) {
return null;
}
nextEnvelope = this.queuedEnvelopes.poll();
}
// schedule events as pending, because events come always after the buffer!
List<AbstractEvent> events = (List<AbstractEvent>) nextEnvelope.deserializeEvents();
Iterator<AbstractEvent> eventsIt = events.iterator();
if (eventsIt.hasNext()) {
this.pendingEvents = eventsIt;
}
// get the buffer, if there is one
if (nextEnvelope.getBuffer() != null) {
return new BufferOrEvent(nextEnvelope.getBuffer());
}
else if (this.pendingEvents != null) {
// if the field is not null, it must always have a next value!
BufferOrEvent next = new BufferOrEvent(this.pendingEvents.next());
if (!this.pendingEvents.hasNext()) {
this.pendingEvents = null;
}
return next;
}
else {
// no buffer and no events, this should be an error
throw new IOException("Received an envelope with neither data nor events.");
}
}
public void transferEventToOutputChannel(AbstractEvent event) throws IOException, InterruptedException {
Envelope ephemeralEnvelope = new Envelope(0, getJobID(), getID());
ephemeralEnvelope.serializeEventList(Arrays.asList(event));
this.envelopeDispatcher.dispatchFromInputChannel(ephemeralEnvelope);
}
}