Package org.jboss.cache.loader.s3

Source Code of org.jboss.cache.loader.s3.S3CacheLoader

/*
* JBoss, Home of Professional Open Source.
* Copyright 2000 - 2008, Red Hat Middleware LLC, and individual contributors
* as indicated by the @author tags. See the copyright.txt file in the
* distribution for a full listing of individual contributors.
*
* This is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this software; if not, write to the Free
* Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
* 02110-1301 USA, or see the FSF site: http://www.fsf.org.
*/
package org.jboss.cache.loader.s3;

import net.jcip.annotations.ThreadSafe;
import net.noderunner.amazon.s3.Bucket;
import net.noderunner.amazon.s3.Connection;
import net.noderunner.amazon.s3.Entry;
import net.noderunner.amazon.s3.GetStreamResponse;
import net.noderunner.amazon.s3.ListResponse;
import net.noderunner.amazon.s3.Response;
import net.noderunner.amazon.s3.S3Object;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.cache.Fqn;
import org.jboss.cache.config.CacheLoaderConfig.IndividualCacheLoaderConfig;
import org.jboss.cache.loader.AbstractCacheLoader;

import java.io.BufferedInputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;


/**
* Uses the Amazon S3 service for storage.
* See http://aws.amazon.com/ for information.
* Does not support transaction isolation.
* <p/>
* Data is stored in a single bucket location.
* The FQN comprises the key of the storage, the data the data itself.
* <p/>
* Internal structure:
* <pre>
* A/
* B/_rootchild
* C/_rootchild/_child1
* C/_rootchild/_child2
* C/_rootchild/_child3
* B/_root2
* </pre>
* The FQN component type is either prefixed with a _ for String, or a primitive type prefix.
* <p/>
* All put and many remove operations require fetching and merging data before storing data,
* which increases latency. This fetching can be turned off. See {@link S3LoaderConfig#getOptimize()}.
* <p/>
* Parent nodes are added to the store as needed.
* For example, when doing a put("/a/b/c"), the nodes "/a/b" and "/a" are created
* if they do not exist. To prevent unnecessary checks of the store,
* a local cache is kept of these "parent nodes". With multiple sites removing
* parent nodes, this can potentially need to inconsistencies. To disable caching,
* set {@link S3LoaderConfig#setParentCache} to false.
* <p/>
*
* @author Elias Ross
* @version $Id: JdbmCacheLoader.java 4298 2007-08-15 18:30:00Z genman $
*/
@ThreadSafe
@SuppressWarnings("unchecked")
public class S3CacheLoader extends AbstractCacheLoader
{
   private static final Log log = LogFactory.getLog(S3CacheLoader.class);
   private static final boolean trace = log.isTraceEnabled();

   /**
    * Max number of dummy parent nodes to cache.
    */
   private static final int PARENT_CACHE_SIZE = 100;

   /**
    * Configuration.
    */
   private S3LoaderConfig config;

   /**
    * Unit separator.
    */
   private final char SEP = Fqn.SEPARATOR.charAt(0);

   /**
    * Zero depth prefix.
    */
   private final char DEPTH_0 = 'A';

   /**
    * Stateless connection; thread safe.
    */
   private Connection connection;

   /**
    * Map classes to characters.
    */
   private static Map<Class<?>, Character> prefix = new HashMap<Class<?>, Character>();

   static
   {
      prefix.put(Byte.class, 'B');
      prefix.put(Character.class, 'C');
      prefix.put(Double.class, 'D');
      prefix.put(Float.class, 'F');
      prefix.put(Integer.class, 'I');
      prefix.put(Long.class, 'J');
      prefix.put(Short.class, 'S');
      prefix.put(Boolean.class, 'Z');
   }

   /**
    * Empty parent nodes whose existence is cached.
    * Empty parents are required to ensure {@link #getChildrenNames(Fqn)}
    * and recursive {@link #remove(Fqn)} work correctly.
    */
   private Set<Fqn> parents = Collections.synchronizedSet(new HashSet<Fqn>());

   /**
    * Empty HashMap, serialized, lazy created.
    */
   private S3Object dummyObj;

   /**
    * This cache loader is stateless, but as part of initialization access the service.
    * Creates a new bucket, if necessary.
    */
   @Override
   public void start() throws Exception
   {
      log.debug("Starting");
      try
      {
         this.connection = config.getConnection();
         Response create = connection.create(getBucket(), config.getLocation());
         if (!create.isOk())
            throw new S3Exception("Unable to create bucket: " + create);
         log.info("S3 accessed successfully. Bucket created: " + create);
      }
      catch (Exception e)
      {
         destroy();
         throw e;
      }
   }

   /**
    * Closes the connection; shuts down the HTTP connection pool.
    */
   @Override
   public void stop()
   {
      log.debug("stop");
      connection.shutdown();
   }

   /**
    * Sets the configuration string for this cache loader.
    */
   public void setConfig(IndividualCacheLoaderConfig base)
   {
      if (base instanceof S3LoaderConfig)
      {
         this.config = (S3LoaderConfig) base;
      }
      else
      {
         config = new S3LoaderConfig(base);
      }

      if (trace)
         log.trace("config=" + config);
   }

   public IndividualCacheLoaderConfig getConfig()
   {
      return config;
   }

   private String key(Fqn fqn)
   {
      return key(fqn.size(), fqn).toString();
   }

   private String children(Fqn fqn)
   {
      return key(fqn.size() + 1, fqn).append(SEP).toString();
   }

   private StringBuilder key(int depth, Fqn fqn)
   {
      StringBuilder sb = new StringBuilder();
      List l = fqn.peekElements();
      sb.append((char) (DEPTH_0 + depth));
      for (Object o : l)
      {
         sb.append(SEP);
         if (o == null)
            sb.append("_null");
         else if (o instanceof String)
            sb.append("_").append(o);
         else
         {
            // TODO
            Character c = prefix.get(o.getClass());
            if (c == null)
               throw new IllegalArgumentException("not supported " + o.getClass());
            sb.append(c.charValue()).append(o);
         }
      }
      return sb;
   }

   /**
    * Returns an unmodifiable set of relative children names, or
    * returns null if the parent node is not found or if no children are found.
    */
   public Set<String> getChildrenNames(Fqn name) throws Exception
   {
      String children = children(name);
      ListResponse response = connection.list(getBucket(), children);
      if (trace)
      {
         log.trace("getChildrenNames " + name + " response=" + response);
      }
      if (response.isNotFound())
         return null;
      if (!response.isOk())
         throw new Exception("List failed " + response);

      Set<String> set = new HashSet<String>();
      for (Entry e : response.getEntries())
      {
         // TODO decode prefix
         set.add(e.getKey().substring(children.length() + 1));
      }

      if (set.isEmpty())
      {
         return null;
      }

      return Collections.unmodifiableSet(set);
   }

   /**
    * Returns a map containing all key-value pairs for the given FQN, or null
    * if the node is not present.
    */
   public Map get(Fqn name) throws Exception
   {
      GetStreamResponse response = connection.getStream(getBucket(), key(name));
      try
      {
         if (trace)
         {
            log.trace("get " + name + " response=" + response);
         }

         if (response.isNotFound())
            return null;
         if (!response.isOk())
            throw new S3Exception("get failed " + response);

         BufferedInputStream is = new BufferedInputStream(response.getInputStream());
         Map map = (Map) getMarshaller().objectFromStream(is);
         response.release();
         return map;
      }
      finally
      {
         response.release();
      }
   }

   private Bucket getBucket()
   {
      return config.getBucket();
   }

   /**
    * Returns whether the given node exists.
    */
   public boolean exists(Fqn name) throws Exception
   {
      Response response = connection.head(getBucket(), key(name));
      if (trace)
      {
         log.trace("exists " + name + " response=" + response);
      }
      return response.isOk();
   }

   private S3Object wrap(Map map) throws Exception
   {
      byte[] b = getMarshaller().objectToByteBuffer(map);
      return new S3Object(b);
   }

   /**
    * Stores a single FQN-key-value record.
    * This is slow, so avoid this method.
    */
   public Object put(Fqn name, Object key, Object value) throws Exception
   {
      Map map = get(name);
      Object oldValue;
      if (map != null)
      {
         oldValue = map.put(key, value);
      }
      else
      {
         map = new HashMap(Collections.singletonMap(key, value));
         oldValue = null;
      }
      put0(name, map);
      return oldValue;
   }

   /**
    * Puts by replacing all the contents of the node.
    */
   private void put0(Fqn name, Map map) throws Exception
   {
      put0(name, wrap(map));
   }

   private void put0(Fqn name, S3Object obj) throws Exception
   {
      Response response = connection.put(getBucket(), key(name), obj);
      if (trace)
      {
         log.trace("put " + name + " obj=" + obj + " response=" + response);
      }
      ensureParent(name);
      if (!response.isOk())
         throw new S3Exception("Put failed " + response);
   }

   private S3Object getDummy() throws Exception
   {
      if (dummyObj != null)
         return dummyObj;
      return dummyObj = wrap(new HashMap(0));
   }

   /**
    * Ensures a parent node exists.
    * Calls recursively to initialize parents as necessary.
    */
   private void ensureParent(Fqn name) throws Exception
   {
      if (name.size() <= 1)
         return;
      Fqn parent = name.getParent();
      boolean cache = config.getParentCache();
      if (cache && parents.contains(parent))
         return;
      // potential race condition between exists and put
      if (!exists(parent))
         put0(parent, getDummy());
      if (cache)
      {
         parents.add(parent);
         if (parents.size() > PARENT_CACHE_SIZE)
         {
            parents.clear();
         }
      }
      ensureParent(parent);
   }

   /**
    * Removes a key from an FQN.
    * Not very fast.
    */
   public Object remove(Fqn name, Object key) throws Exception
   {
      Map map = get(name);
      Object oldValue;
      if (map != null)
      {
         oldValue = map.remove(key);
      }
      else
      {
         oldValue = null;
      }
      put0(name, map);
      return oldValue;
   }

   /**
    * Stores a map of key-values for a given FQN, but does not delete existing
    * key-value pairs (that is, it does not erase).
    */
   public void put(Fqn name, Map<Object, Object> values) throws Exception
   {
      if (values == null)
         values = Collections.emptyMap();
      put0(name, values);
   }

   /**
    * Deletes the node for a given FQN and all its descendant nodes.
    */
   public void remove(Fqn name) throws Exception
   {
      /*
      if (name.isRoot())
      {
        log.trace("optimized delete");
       connection.delete(getBucket()).assertOk();
       connection.create(getBucket()).assertOk();
        log.trace("done");
       return;
      }
      */
      Set<String> children = getChildrenNames(name);
      if (children != null)
      {
         log.trace("remove children: " + children);
         for (String child : children)
         {
            remove(Fqn.fromRelativeElements(name, child));
         }
      }
      Response response = connection.delete(getBucket(), key(name));
      if (trace)
      {
         log.trace("delete " + name + " response=" + response);
      }
      if (!response.isOk() && !response.isNotFound())
         throw new S3Exception("delete failed " + response);
      parents.remove(name);
   }

   /**
    * Clears the map for the given node, but does not remove the node.
    */
   public void removeData(Fqn name) throws Exception
   {
      put0(name, getDummy());
   }

}
TOP

Related Classes of org.jboss.cache.loader.s3.S3CacheLoader

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.