Package eu.stratosphere.runtime.io

Examples of eu.stratosphere.runtime.io.Buffer

75
76
77
78
79
80
81
    this.outputs = new LinkedList<OutputGate>();

    this.memManager = new DefaultMemoryManager(memorySize);
    this.ioManager = new IOManager(System.getProperty("java.io.tmpdir"));
    this.inputSplitProvider = inputSplitProvider;
    this.mockBuffer = new Buffer(new MemorySegment(new byte[bufferSize]), bufferSize, null);
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
    // --------------------------------------------------------------------
    // (3) buffer (var length)
    // --------------------------------------------------------------------
    if (env.getBuffer() != null) {
      Buffer buffer = env.getBuffer();
      out.writeBytes(buffer.getMemorySegment().wrap(0, buffer.size()));

      // Recycle the buffer from OUR buffer pool after everything has been
      // copied to Nettys buffer space.
      buffer.recycleBuffer();
    }
  }
View Full Code Here
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    // --------------------------------------------------------------------
    // (a) request a buffer from OUR pool
    if (this.currentBufferRequestSize > 0) {
      JobID jobId = this.currentEnvelope.getJobID();
      ChannelID sourceId = this.currentEnvelope.getSource();
      Buffer buffer = requestBufferForTarget(jobId, sourceId, this.currentBufferRequestSize);

      if (buffer == null) {
        return DecoderState.NO_BUFFER_AVAILABLE;
      }
      else {
        this.currentEnvelope.setBuffer(buffer);
        this.currentDataBuffer = buffer.getMemorySegment().wrap(0, this.currentBufferRequestSize);
        this.currentBufferRequestSize = 0;
      }
    }

    // (b) copy data to OUR buffer
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
   * This task should be executed by the IO thread to ensure safe access to the staged buffer.
   */
  private class BufferAvailabilityChangedTask implements Runnable {
    @Override
    public void run() {
      Buffer availableBuffer = bufferBroker.poll();
      if (availableBuffer == null) {
        throw new IllegalStateException("The BufferAvailabilityChangedTask" +
            "should only be executed when a Buffer has been offered" +
            "to the Buffer broker (after becoming available).");
      }

      // This alters the state of the last `decodeEnvelope(ByteBuf)`
      // call to set the buffer, which has become available again
      availableBuffer.limitSize(currentBufferRequestSize);
      currentEnvelope.setBuffer(availableBuffer);
      currentDataBuffer = availableBuffer.getMemorySegment().wrap(0, InboundEnvelopeDecoder.this.currentBufferRequestSize);
      currentBufferRequestSize = 0;

      stagedBuffer.release();

      try {
        if (decodeBuffer(stagedBuffer, channelHandlerContext)) {
          stagedBuffer = null;
          channelHandlerContext.channel().config().setAutoRead(true);
          if (LOG.isDebugEnabled()) {
            LOG.debug(String.format("Set channel %s auto read to true.", channelHandlerContext.channel()));
          }
        }
      } catch (IOException e) {
        availableBuffer.recycleBuffer();
      }
    }
View Full Code Here
47
48
49
50
51
52
53
54
  public Buffer requestBuffer(int sizeOfBuffer) throws IOException {
    if (rnd.nextFloat() < this.probabilityForNoneAvailable) {
      return null;
    } else {
      MemorySegment segment = new MemorySegment(new byte[this.sizeOfMemorySegments]);
      return new Buffer(segment, sizeOfBuffer, this.recycler);
    }
  }
View Full Code Here
54
55
56
57
58
59
60
  }

  @Override
  public Buffer requestBufferBlocking(int sizeOfBuffer) throws IOException, InterruptedException {
    MemorySegment segment = new MemorySegment(new byte[this.sizeOfMemorySegments]);
    return new Buffer(segment, sizeOfBuffer, this.recycler);
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    final int SERIALIZATION_OVERHEAD = 4; // length encoding

    final RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final RecordDeserializer<SerializationTestType> deserializer = new AdaptiveSpanningRecordDeserializer<SerializationTestType>();

    final Buffer buffer = new Buffer(new MemorySegment(new byte[segmentSize]), segmentSize, null);

    final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<SerializationTestType>();

    // -------------------------------------------------------------------------------------------------------------
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  @Test
  public void testEncodedSizeAndBufferRecycling() {
    final ByteBuffer events = ByteBuffer.allocate(MAX_EVENTS_SIZE);
    final MemorySegment segment = new MemorySegment(new byte[MAX_BUFFER_SIZE]);

    final Buffer buffer = mock(Buffer.class);
    when(buffer.getMemorySegment()).thenReturn(segment);

    final EmbeddedChannel channel = new EmbeddedChannel(new OutboundEnvelopeEncoder());

    int numBuffers = 0;
    for (int i = 0; i < NUM_RANDOM_ENVELOPES; i++) {
      Envelope env = new Envelope(i, new JobID(), new ChannelID());
      int expectedEncodedMsgSize = OutboundEnvelopeEncoder.HEADER_SIZE;

      if (random.nextBoolean()) {
        int eventsSize = random.nextInt(MAX_EVENTS_SIZE + 1);
        expectedEncodedMsgSize += eventsSize;

        events.clear();
        events.limit(eventsSize);

        env.setEventsSerialized(events);
      }

      if (random.nextBoolean()) {
        numBuffers++;

        int bufferSize = random.nextInt(MAX_BUFFER_SIZE + 1);
        when(buffer.size()).thenReturn(bufferSize);
        env.setBuffer(buffer);

        expectedEncodedMsgSize += bufferSize;
      }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    Assert.assertEquals(0, bufferPool.numRequestedBuffers());

    // this request-recycle cycle should only take a single buffer out of
    // the GlobalBufferPool as it is recycled over and over again
    for (int numRequested = 0; numRequested < NUM_BUFFERS; numRequested++) {
      Buffer buffer = bufferPool.requestBuffer(BUFFER_SIZE);

      Assert.assertEquals(BUFFER_SIZE, buffer.size());

      Assert.assertEquals("Expected single buffer request in buffer pool.",
          1, bufferPool.numRequestedBuffers());
      Assert.assertEquals("Expected no available buffer in buffer pool.",
          0, bufferPool.numAvailableBuffers());

      buffer.recycleBuffer();

      Assert.assertEquals("Expected single available buffer after recycle.",
          1, bufferPool.numAvailableBuffers());
    }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
  @Test
  public void testSingleConsumerNonBlockingRequestSmall() throws IOException {
    final LocalBufferPool bufferPool = new LocalBufferPool(GLOBAL_BUFFER_POOL, NUM_BUFFERS);

    // request smaller buffer and verify size
    Buffer buffer = bufferPool.requestBuffer(BUFFER_SIZE / 2);

    Assert.assertEquals(BUFFER_SIZE / 2, buffer.size());

    buffer.recycleBuffer();

    bufferPool.destroy();
  }
View Full Code Here
TOP

Related Classes of eu.stratosphere.runtime.io.Buffer

  • eu.stratosphere.nephele.util.TestBufferProvider
  • eu.stratosphere.pact.runtime.test.util.MockEnvironment
  • eu.stratosphere.runtime.io.api.RecordWriter
  • eu.stratosphere.runtime.io.channels.InputChannel
  • eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPool
  • eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPoolTest
  • eu.stratosphere.runtime.io.network.bufferprovider.LocalBufferPoolTest$RecyclingBufferAvailableAnswer
  • eu.stratosphere.runtime.io.network.ChannelManager
  • eu.stratosphere.runtime.io.network.netty.InboundEnvelopeDecoder
  • eu.stratosphere.runtime.io.network.netty.InboundEnvelopeDecoder$BufferAvailabilityChangedTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.