Package org.apache.nutch.util

Examples of org.apache.nutch.util.ObjectCache


  public URLNormalizers(Configuration conf, String scope) {
    this.conf = conf;
    this.extensionPoint = PluginRepository.get(conf).getExtensionPoint(
            URLNormalizer.X_POINT_ID);
    ObjectCache objectCache = ObjectCache.get(conf);
   
    if (this.extensionPoint == null) {
      throw new RuntimeException("x point " + URLNormalizer.X_POINT_ID
              + " not found.");
    }

    normalizers = (URLNormalizer[])objectCache.getObject(URLNormalizer.X_POINT_ID + "_" + scope);
    if (normalizers == null) {
      normalizers = getURLNormalizers(scope);
    }
    if (normalizers == EMPTY_NORMALIZERS) {
      normalizers = (URLNormalizer[])objectCache.getObject(URLNormalizer.X_POINT_ID + "_" + SCOPE_DEFAULT);
      if (normalizers == null) {
        normalizers = getURLNormalizers(SCOPE_DEFAULT);
      }
    }
   
View Full Code Here


   *         scope.
   * @throws PluginRuntimeException
   */
  URLNormalizer[] getURLNormalizers(String scope) {
    List<Extension> extensions = getExtensions(scope);
    ObjectCache objectCache = ObjectCache.get(conf);
   
    if (extensions == EMPTY_EXTENSION_LIST) {
      return EMPTY_NORMALIZERS;
    }
   
    List<URLNormalizer> normalizers = new Vector<URLNormalizer>(extensions.size());

    Iterator<Extension> it = extensions.iterator();
    while (it.hasNext()) {
      Extension ext = it.next();
      URLNormalizer normalizer = null;
      try {
        // check to see if we've cached this URLNormalizer instance yet
        normalizer = (URLNormalizer) objectCache.getObject(ext.getId());
        if (normalizer == null) {
          // go ahead and instantiate it and then cache it
          normalizer = (URLNormalizer) ext.getExtensionInstance();
          objectCache.setObject(ext.getId(), normalizer);
        }
        normalizers.add(normalizer);
      } catch (PluginRuntimeException e) {
        e.printStackTrace();
        LOG.warn("URLNormalizers:PluginRuntimeException when "
View Full Code Here

   *         empty list.
   * @throws PluginRuntimeException
   */
  @SuppressWarnings("unchecked")
  private List<Extension> getExtensions(String scope) {
    ObjectCache objectCache = ObjectCache.get(conf);
    List<Extension> extensions =
      (List<Extension>) objectCache.getObject(URLNormalizer.X_POINT_ID + "_x_"
                                                + scope);

    // Just compare the reference:
    // if this is the empty list, we know we will find no extension.
    if (extensions == EMPTY_EXTENSION_LIST) {
      return EMPTY_EXTENSION_LIST;
    }

    if (extensions == null) {
      extensions = findExtensions(scope);
      if (extensions != null) {
        objectCache.setObject(URLNormalizer.X_POINT_ID + "_x_" + scope, extensions);
      } else {
        // Put the empty extension list into cache
        // to remember we don't know any related extension.
        objectCache.setObject(URLNormalizer.X_POINT_ID + "_x_" + scope, EMPTY_EXTENSION_LIST);
        extensions = EMPTY_EXTENSION_LIST;
      }
    }
    return extensions;
  }
View Full Code Here

  private SignatureFactory() {}                   // no public ctor

  /** Return the default Signature implementation. */
  public synchronized static Signature getSignature(Configuration conf) {
    String clazz = conf.get("db.signature.class", MD5Signature.class.getName());
    ObjectCache objectCache = ObjectCache.get(conf);
    Signature impl = (Signature)objectCache.getObject(clazz);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Using Signature impl: " + clazz);
        }
        Class<?> implClass = Class.forName(clazz);
        impl = (Signature)implClass.newInstance();
        impl.setConf(conf);
        objectCache.setObject(clazz, impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create " + clazz, e);
      }
    }
    return impl;
View Full Code Here

   * @throws ProtocolNotFound
   *           when Protocol can not be found for urlString
   */
  public synchronized Protocol getProtocol(String urlString)
      throws ProtocolNotFound {
    ObjectCache objectCache = ObjectCache.get(conf);
    try {
      URL url = new URL(urlString);
      String protocolName = url.getProtocol();
      if (protocolName == null)
        throw new ProtocolNotFound(urlString);

      String cacheId = Protocol.X_POINT_ID + protocolName;
      Protocol protocol = (Protocol) objectCache.getObject(cacheId);
      if (protocol != null) {
        return protocol;
      }

      Extension extension = findExtension(protocolName);
      if (extension == null) {
        throw new ProtocolNotFound(protocolName);
      }

      protocol = (Protocol) extension.getExtensionInstance();
      objectCache.setObject(cacheId, protocol);
      return protocol;
    } catch (MalformedURLException e) {
      throw new ProtocolNotFound(urlString, e.toString());
    } catch (PluginRuntimeException e) {
      throw new ProtocolNotFound(urlString, e.toString());
View Full Code Here

      .getLogger(IndexWriters.class);

  private IndexWriter[] indexWriters;

  public IndexWriters(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    synchronized (objectCache) {
      this.indexWriters = (IndexWriter[]) objectCache
          .getObject(IndexWriter.class.getName());
      if (this.indexWriters == null) {
        try {
          ExtensionPoint point = PluginRepository.get(conf)
              .getExtensionPoint(IndexWriter.X_POINT_ID);
          if (point == null)
            throw new RuntimeException(IndexWriter.X_POINT_ID
                + " not found.");
          Extension[] extensions = point.getExtensions();
          HashMap<String, IndexWriter> indexerMap = new HashMap<String, IndexWriter>();
          for (int i = 0; i < extensions.length; i++) {
            Extension extension = extensions[i];
            IndexWriter writer = (IndexWriter) extension
                .getExtensionInstance();
            LOG.info("Adding " + writer.getClass().getName());
            if (!indexerMap.containsKey(writer.getClass().getName())) {
              indexerMap.put(writer.getClass().getName(), writer);
            }
          }
          objectCache.setObject(IndexWriter.class.getName(), indexerMap
              .values().toArray(new IndexWriter[0]));
        } catch (PluginRuntimeException e) {
          throw new RuntimeException(e);
        }
        this.indexWriters = (IndexWriter[]) objectCache
            .getObject(IndexWriter.class.getName());
      }
    }
  }
View Full Code Here

    }
  }
 
  public static CollectionManager getCollectionManager(Configuration conf) {
    String key = "collectionmanager";
    ObjectCache objectCache = ObjectCache.get(conf);
    CollectionManager impl = (CollectionManager)objectCache.getObject(key);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Instantiating CollectionManager");
        }
        impl=new CollectionManager(conf);
        objectCache.setObject(key,impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create CollectionManager", e);
      }
    }
    return impl;
View Full Code Here

   * @return array of plugin instances
   */
  public synchronized Object[] getOrderedPlugins(Class<?> clazz, String xPointId,
      String orderProperty) {
    Object[] filters;
    ObjectCache objectCache = ObjectCache.get(conf);
    filters = (Object[]) objectCache.getObject(clazz.getName());

    if (filters == null) {
      String order = conf.get(orderProperty);
      List<String> orderOfFilters = new ArrayList<String>();
      boolean userDefinedOrder = false;
      if (order != null && !order.trim().isEmpty()) {
        orderOfFilters = Arrays.asList(order.trim().split("\\s+"));
        userDefinedOrder = true;
      }

      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            xPointId);
        if (point == null)
          throw new RuntimeException(xPointId + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, Object> filterMap = new HashMap<String, Object>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          Object filter = extension.getExtensionInstance();
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
            if (!userDefinedOrder)
              orderOfFilters.add(filter.getClass().getName());
          }
        }
        List<Object> sorted = new ArrayList<Object>();
        for (String orderedFilter : orderOfFilters) {
          Object f = filterMap.get(orderedFilter);
          if (f == null) {
            LOG.error(clazz.getSimpleName() + " : " + orderedFilter
                + " declared in configuration property " + orderProperty
                + " but not found in an active plugin - ignoring.");
            continue;
          }
          sorted.add(f);
        }
        Object[] filter = (Object[]) Array.newInstance(clazz, sorted.size());
        for (int i = 0; i < sorted.size(); i++) {
          filter[i] = sorted.get(i);
          if (LOG.isTraceEnabled()) {
            LOG.trace(clazz.getSimpleName() + " : filters[" + i + "] = "
                + filter[i].getClass());
          }
        }
        objectCache.setObject(clazz.getName(), filter);
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }

      filters = (Object[]) objectCache.getObject(clazz.getName());
    }
    return filters;
  }
View Full Code Here

  private ExtensionPoint extensionPoint;
  private ParsePluginList parsePluginList;

  public ParserFactory(Configuration conf) {
    this.conf = conf;
    ObjectCache objectCache = ObjectCache.get(conf);
    this.extensionPoint = PluginRepository.get(conf).getExtensionPoint(Parser.X_POINT_ID);
    this.parsePluginList = (ParsePluginList)objectCache.getObject(ParsePluginList.class.getName());
   
    if (this.parsePluginList == null) {
      this.parsePluginList = new ParsePluginsReader().parse(conf);
      objectCache.setObject(ParsePluginList.class.getName(), this.parsePluginList);
    }

    if (this.extensionPoint == null) {
      throw new RuntimeException("x point " + Parser.X_POINT_ID + " not found.");
    }
View Full Code Here

  throws ParserNotFound {
   
    List<Parser> parsers = null;
    List<Extension> parserExts = null;
   
    ObjectCache objectCache = ObjectCache.get(conf);
   
    // TODO once the MimeTypes is available
    // parsers = getExtensions(MimeUtils.map(contentType));
    // if (parsers != null) {
    //   return parsers;
    // }
    // Last Chance: Guess content-type from file url...
    // parsers = getExtensions(MimeUtils.getMimeType(url));

    parserExts = getExtensions(contentType);
    if (parserExts == null) {
      throw new ParserNotFound(url, contentType);
    }

    parsers = new Vector<Parser>(parserExts.size());
    for (Iterator<Extension> i = parserExts.iterator(); i.hasNext(); ){
      Extension ext = i.next();
      Parser p = null;
      try {
        //check to see if we've cached this parser instance yet
        p = (Parser) objectCache.getObject(ext.getId());
        if (p == null) {
          // go ahead and instantiate it and then cache it
          p = (Parser) ext.getExtensionInstance();
          objectCache.setObject(ext.getId(),p);
        }
        parsers.add(p);
      } catch (PluginRuntimeException e) {
        if (LOG.isWarnEnabled()) {
          LOG.warn("ParserFactory:PluginRuntimeException when "
View Full Code Here

TOP

Related Classes of org.apache.nutch.util.ObjectCache

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.