Examples of com.shop.util.chunked.ChunkedByteArray

com.shop.util.chunked.ChunkedByteArray
Abstracts an unbounded array that is internally broken into chunks. This avoids allocating large contiguous byte arrays which is very inefficient in Java. The ChunkedByteArray contrasts with JDK classes such as ByteBuffer, byte[], ByteArrayOutputStream, etc. As those JDK classes get larger, they get more inefficient. In particular, ByteArrayOutputStream, must copy old data as it grows. Further, the memory allocator in Java is not optimized for large objects. Smaller objects get allocated in the young heap and are allocated very fast. The ChunkedByteArray takes advantage of this by only allocating relatively small objects but maintaining an arbitrarily large logical size to clients of the class.

The default chunk size is 32K. This can be changed by setting the system property ChunkedByteArrayDefaultSize to any number of bytes you like.

Note that this implementation is not synchronized. If multiple threads access a ChunkedByteArray instance concurrently, and at least one of the threads modifies the array structurally (by calling {@link #clear()} or one of theappend(...) methods), it must be synchronized externally.
@since 1.3 1/30/08 rewrote several of the append() methods. Added an equals() implementation.
1.2 12/24/07 reworked offset calcs so that reads are thread-safe. I was originally worried about making an allocation to do it. But, as Brian Goetz says, allocations can be as little as 10 instructions. My simple testing shows that the allocation in {@link #calcTempOffsets(int)} doesn't alter performance.
1.1 12/23/07 append(InputStream in, Integer length) was incorrect if length was non-null. The read from the input stream could ask for too many bytes. Also, updated Javadoc.
@author Jordan Zimmerman <jordan@jordanzimmerman.com> @see ChunkedByteArrayInputStream @see ChunkedByteArrayOutputStream

  {
    key = filterKey(key);


    try
    {
      ChunkedByteArray  bytes = null;
      
      String         commandName = SCSetOfCommands.getCommandName(ignoreTTL ? SCCommandGetObjectIgnoreTTL.class : SCCommandGetObject.class);
      fClient.send(commandName);
      fClient.send(key);
      fClient.flush();


      int      size = safeParseInt(fClient.readLine());
      if ( size > 0 )
      {
        bytes = fClient.readBytes(size);
      }


      if ( (bytes == null) || (bytes.size() == 0) )
      {
        return null;
      }


      return bytes;

View Full Code Here

      if ( (activePendingPut != null) && activePendingPut.key.equals(key) )
      {
        return activePendingPut.spec;
      }


      ChunkedByteArray    data = null;


      CCDB2IndexEntry     entry = fIndex.get(key);
      if ( entry != null )
      {
        synchronized(entry)
        {
          if ( entry.address == CCDB2IndexEntry.NOT_EXISTS_ADDRESS )
          {
            if ( wasDeleted != null )
            {
              wasDeleted.set(true);
            }
          }
          else
          {
            data = (entry.bytesRef != null) ? entry.bytesRef.get() : null;
            if ( data == null )
            {
              if ( entry.address >= CCDB2IndexEntry.MINIMUM_ACTIVE_ADDRESS )
              {
                data = readObject(entry.address);
                if ( data == null )
                {
                  if ( wasDeleted != null )
                  {
                    wasDeleted.set(true);
                  }
                  entry.address = CCDB2IndexEntry.NOT_EXISTS_ADDRESS;
                }
                else
                {
                  data.lock();
                  entry.bytesRef = new SoftReference<ChunkedByteArray>(data);


                  fFromDiskGetQty.incrementAndGet();
                }
              }

View Full Code Here

    fIndexSize.addAndGet(size);
  }


  private void processPut(CCDB2IndexEntry entry, String key, CCDB2DataSpec spec, long[] groupSpecs, boolean addToIndexFile) throws IOException
  {
    ChunkedByteArray     previous = (entry.bytesRef != null) ? entry.bytesRef.get() : null;
    PendingPutRecord    pendingPut = new PendingPutRecord(key, previous, entry, spec, groupSpecs, addToIndexFile);


    // though this is a SoftReference, a hard reference is held by spec.data in the pending record until it's actually written
    entry.bytesRef = new SoftReference<ChunkedByteArray>(spec.data);
    if ( fPendingPutQueue != null )

View Full Code Here

  {
    final RandomAccessFile   file = io.getUnderlyingFile();


    if ( driver.doChunking() && (fObjectSize >= ChunkedByteArray.DEFAULT_CHUNK_SIZE) )
    {
      fObject = new ChunkedByteArray();
      fObject.append
      (
        new InputStream()
        {
          @Override

View Full Code Here

   */
  private void  processPut(PutWrapper wrapper)
  {
    try
    {
      ChunkedByteArray     data = fSerializer.serialize(wrapper.block);
      SCDataSpec spec = new SCDataSpec(data, wrapper.block.getTTL());
      if ( wrapper.withBackup )
      {
        fManager.putWithBackup(wrapper.block.getKey(), spec, wrapper.block.getGroups());
      }

View Full Code Here

  private Object requestObject(SCDataBlock block, long rightNow)
  {
    Object     resultObject = null;
    try
    {
      ChunkedByteArray   data = fManager.get(block.getKey(), block.getIgnoreTTL());
      SCDataBlock     fromManagerBlock = (data != null) ? fSerializer.deserialize(data) : null;
      if ( fromManagerBlock != null )
      {
        boolean     isUseable = checkIsUsable(block, fromManagerBlock, rightNow);
        if ( isUseable )

View Full Code Here

  }


  @Override
  public ChunkedByteArray serialize(SCDataBlock block) throws Exception
  {
    ChunkedByteArray        chunked = new ChunkedByteArray();
    ObjectOutputStream         out = new ObjectOutputStream(new GZIPOutputStream(new ChunkedByteArrayOutputStream(chunked)));


    out.writeInt(FILE_VERSION);
    out.writeInt(block.getVersionNumber());
    out.writeLong(block.getTTL());
    out.writeUTF(block.getKey());


    out.writeObject(block.getObject());


    out.close();


    chunked.lock();
    return chunked;
  }

View Full Code Here

          case OBJECT:
          {
            int    size = sizeFromLine(fClient.readLine());
            if ( size > 0 )
            {
              ChunkedByteArray    bytes = fClient.readBytes(size);
              builder.addNextObject(bytes);
            }
            break;
          }

View Full Code Here

      }


      @Override
      public void executeCommand(SCServer server, SCConnection connection) throws Exception
      {
        ChunkedByteArray     data = server.get(fKey, fIgnoreTTL);
        connection.sendObject((data != null) ? data : null);
      }


      private String      fKey = "";
    };

View Full Code Here


  @Override
  public ChunkedByteArray readBytes(int size) throws IOException
  {
    boolean          eof = false;
    ChunkedByteArray    bytes = (size < ChunkedByteArray.DEFAULT_CHUNK_SIZE) ? new ChunkedByteArray(size) : new ChunkedByteArray();
    while ( size-- > 0 )
    {
      int    i = fIn.read();
      if ( i < 0 )
      {
        eof = true;
        break;
      }
      bytes.append((byte)(i & 0xff));
      updateLastReadTicks();
    }


    if ( eof )
    {
      if ( bytes.size() == 0 )
      {
        return null;
      }
      throw new EOFException();
    }

View Full Code Here

TOP

Related Classes of com.shop.util.chunked.ChunkedByteArray

com.shop.cache.api.client.main.DefaultSerializer

com.shop.cache.api.client.main.SCCache

com.shop.cache.api.commands.SCCommandGetObjectBase

com.shop.cache.imp.client.ImpSCClient

com.shop.cache.imp.server.ImpSCServerConnection

com.shop.util.ccdb2.CCDB2Instance

com.shop.util.ccdb2.CCDB2Record

com.shop.util.generic.GenericIOClientImpl

java.io.EOFException

java.util.Random

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.