Package eu.stratosphere.core.memory

Examples of eu.stratosphere.core.memory.MemorySegment


   
    ensureNumBuffersReturned(numSegs);
   
    // go over all segments that are part of the table
    for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
      final MemorySegment seg = getNextBuffer();
     
      // go over all buckets in the segment
      for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
        final int bucketOffset = k * HASH_BUCKET_SIZE; 
       
        // compute the partition that the bucket corresponds to
        final byte partition = assignPartition(bucket, numPartitions);
       
        // initialize the header fields
        seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
        seg.put(bucketOffset + HEADER_STATUS_OFFSET, BUCKET_STATUS_IN_MEMORY);
        seg.putShort(bucketOffset + HEADER_COUNT_OFFSET, (short) 0);
        seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
      }
     
      table[i] = seg;
    }
    this.buckets = table;
View Full Code Here


    // spill the partition
    int numBuffersFreed = p.spillPartition(this.availableMemory, this.ioManager,
                    this.currentEnumerator.next(), this.writeBehindBuffers);
    this.writeBehindBuffersAvailable += numBuffersFreed;
    // grab as many buffers as are available directly
    MemorySegment currBuff = null;
    while (this.writeBehindBuffersAvailable > 0 && (currBuff = this.writeBehindBuffers.poll()) != null) {
      this.availableMemory.add(currBuff);
      this.writeBehindBuffersAvailable--;
    }
    return largestPartNum;
View Full Code Here

    }
   
    // check if there are write behind buffers that actually are to be used for the hash table
    if (this.writeBehindBuffersAvailable > 0) {
      // grab at least one, no matter what
      MemorySegment toReturn;
      try {
        toReturn = this.writeBehindBuffers.take();
      }
      catch (InterruptedException iex) {
        throw new RuntimeException("Hybrid Hash Join was interrupted while taking a buffer.");
      }
      this.writeBehindBuffersAvailable--;
     
      // grab as many more buffers as are available directly
      MemorySegment currBuff = null;
      while (this.writeBehindBuffersAvailable > 0 && (currBuff = this.writeBehindBuffers.poll()) != null) {
        this.availableMemory.add(currBuff);
        this.writeBehindBuffersAvailable--;
      }
      return toReturn;
View Full Code Here

  }
 

  @Override
  public MemorySegment nextSegment() {
    final MemorySegment seg = getNextBuffer();
    if (seg == null) {
      try {
        spillPartition();
      } catch (IOException ioex) {
        throw new RuntimeException("Error spilling Hash Join Partition" + (ioex.getMessage() == null ?
          "." : ": " + ioex.getMessage()), ioex);
      }
     
      MemorySegment fromSpill = getNextBuffer();
      if (fromSpill == null) {
        throw new RuntimeException("BUG in Hybrid Hash Join: Spilling did not free a buffer.");
      } else {
        return fromSpill;
      }
View Full Code Here

      final int searchHashCode = hash(this.probeTypeComparator.hash(probeSideRecord));
     
      final int posHashCode = searchHashCode % numBuckets;
     
      // get the bucket for the given hash code
      MemorySegment bucket = buckets[posHashCode >> bucketsPerSegmentBits];
      int bucketInSegmentOffset = (posHashCode & bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
     
      // get the basic characteristics of the bucket
      final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET);
      final InMemoryPartition<T> partition = partitions.get(partitionNumber);
      final MemorySegment[] overflowSegments = partition.overflowSegments;
     
      this.pairComparator.setReference(probeSideRecord);
     
      int countInSegment = bucket.getInt(bucketInSegmentOffset + HEADER_COUNT_OFFSET);
      int numInSegment = 0;
      int posInSegment = bucketInSegmentOffset + BUCKET_HEADER_LENGTH;

      // loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
      while (true) {
       
        while (numInSegment < countInSegment) {
         
          final int thisCode = bucket.getInt(posInSegment);
          posInSegment += HASH_CODE_LEN;
           
          // check if the hash code matches
          if (thisCode == searchHashCode) {
            // get the pointer to the pair
            final int pointerOffset = bucketInSegmentOffset + BUCKET_POINTER_START_OFFSET + (numInSegment * POINTER_LEN);
            final long pointer = bucket.getLong(pointerOffset);
            numInSegment++;
           
            // deserialize the key to check whether it is really equal, or whether we had only a hash collision
            try {
              partition.readRecordAt(pointer, targetForMatch);
             
              if (this.pairComparator.equalToReference(targetForMatch)) {
                this.partition = partition;
                this.bucket = bucket;
                this.pointerOffsetInBucket = pointerOffset;
                return true;
              }
            }
            catch (IOException e) {
              throw new RuntimeException("Error deserializing record from the hashtable: " + e.getMessage(), e);
            }
          }
          else {
            numInSegment++;
          }
        }
       
        // this segment is done. check if there is another chained bucket
        final long forwardPointer = bucket.getLong(bucketInSegmentOffset + HEADER_FORWARD_OFFSET);
        if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
          return false;
        }
       
        final int overflowSegNum = (int) (forwardPointer >>> 32);
        bucket = overflowSegments[overflowSegNum];
        bucketInSegmentOffset = (int) (forwardPointer & 0xffffffff);
        countInSegment = bucket.getInt(bucketInSegmentOffset + HEADER_COUNT_OFFSET);
        posInSegment = bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
        numInSegment = 0;
      }
    }
View Full Code Here

    this.outputs = new LinkedList<OutputGate>();

    this.memManager = new DefaultMemoryManager(memorySize);
    this.ioManager = new IOManager(System.getProperty("java.io.tmpdir"));
    this.inputSplitProvider = inputSplitProvider;
    this.mockBuffer = new Buffer(new MemorySegment(new byte[bufferSize]), bufferSize, null);
  }
View Full Code Here

   */
  public MemorySegment getNextReturnedSegment() throws IOException
  {
    try {
      while (true) {
        final MemorySegment next = this.returnBuffers.poll(2000, TimeUnit.MILLISECONDS);
        if (next != null) {
          return next;
        } else {
          if (this.closed) {
            throw new IOException("The reader has been asynchronously closed.");
View Full Code Here

    }
    this.closed = true;
   
    // re-collect all memory segments
    ArrayList<MemorySegment> list = this.freeMem;
    final MemorySegment current = getCurrentSegment();
    if (current != null) {
      list.add(current);
    }
    clear();

    // close the writer and gather all segments
    final LinkedBlockingQueue<MemorySegment> queue = this.reader.getReturnQueue();
    this.reader.close();

    while (list.size() < this.numSegments) {
      final MemorySegment m = queue.poll();
      if (m == null) {
        // we get null if the queue is empty. that should not be the case if the reader was properly closed.
        throw new RuntimeException("Bug in ChannelReaderInputView: MemorySegments lost.");
      }
      list.add(m);
View Full Code Here

    if (current != null) {
      sendReadRequest(current);
    }
   
    // get the next segment
    final MemorySegment seg = this.reader.getNextReturnedSegment();
   
    // check the header
    if (seg.getShort(0) != ChannelWriterOutputView.HEADER_MAGIC_NUMBER) {
      throw new IOException("The current block does not belong to a ChannelWriterOutputView / " +
          "ChannelReaderInputView: Wrong magic number.");
    }
    if ( (seg.getShort(ChannelWriterOutputView.HEADER_FLAGS_OFFSET) & ChannelWriterOutputView.FLAG_LAST_BLOCK) != 0) {
      // last block
      this.numRequestsRemaining = 0;
      this.inLastBlock = true;
    }
   
View Full Code Here

    } else {
      this.numSegments = memory.size();
      // load the segments into the queue
      final LinkedBlockingQueue<MemorySegment> queue = writer.getReturnQueue();
      for (int i = memory.size() - 1; i >= 0; --i) {
        final MemorySegment seg = memory.get(i);
        if (seg.size() != segmentSize) {
          throw new IllegalArgumentException("The supplied memory segments are not of the specified size.");
        }
        queue.add(seg);
      }
    }
View Full Code Here

TOP

Related Classes of eu.stratosphere.core.memory.MemorySegment

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.