Package org.apache.nutch.util

Examples of org.apache.nutch.util.ObjectCache


  public Parser getParserById(String id) throws ParserNotFound {

    Extension[] extensions = this.extensionPoint.getExtensions();
    Extension parserExt = null;

    ObjectCache objectCache = ObjectCache.get(conf);
   
    if (id != null) {
      parserExt = getExtension(extensions, id);
    }
    if (parserExt == null) {
      parserExt = getExtensionFromAlias(extensions, id);
    }

    if (parserExt == null) {
      throw new ParserNotFound("No Parser Found for id [" + id + "]");
    }
   
    // first check the cache          
    if (objectCache.getObject(parserExt.getId()) != null) {
      return (Parser) objectCache.getObject(parserExt.getId());

    // if not found in cache, instantiate the Parser   
    } else {
      try {
        Parser p = (Parser) parserExt.getExtensionInstance();
        objectCache.setObject(parserExt.getId(), p);
        return p;
      } catch (PluginRuntimeException e) {
        if (LOG.isWarnEnabled()) {
          LOG.warn("Canno initialize parser " +
                   parserExt.getDescriptor().getPluginId() +
View Full Code Here


   * @return a list of extensions to be used for this contentType.
   *         If none, returns <code>null</code>.
   */
  protected List<Extension> getExtensions(String contentType) {
   
    ObjectCache objectCache = ObjectCache.get(conf);
    // First of all, tries to clean the content-type
    String type = null;
    type = MimeUtil.cleanMimeType(contentType);


    List<Extension> extensions = (List<Extension>) objectCache.getObject(type);

    // Just compare the reference:
    // if this is the empty list, we know we will find no extension.
    if (extensions == EMPTY_EXTENSION_LIST) {
      return null;
    }
   
    if (extensions == null) {
      extensions = findExtensions(type);
      if (extensions != null) {
        objectCache.setObject(type, extensions);
      } else {
        // Put the empty extension list into cache
        // to remember we don't know any related extension.
        objectCache.setObject(type, EMPTY_EXTENSION_LIST);
      }
    }
    return extensions;
  }
View Full Code Here

  private Map<String, String> keyMap = new HashMap<String, String>();
  private Map<String, String> copyMap = new HashMap<String, String>();
  private String uniqueKey = "id";
 
  public static synchronized SolrMappingReader getInstance(Configuration conf) {
    ObjectCache cache = ObjectCache.get(conf);
    SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
    if (instance == null) {
      instance = new SolrMappingReader(conf);
      cache.setObject(SolrMappingReader.class.getName(), instance);
    }
    return instance;
  }
View Full Code Here

 
  public static final String HTMLPARSEFILTER_ORDER = "htmlparsefilter.order";

  public HtmlParseFilters(Configuration conf) {
        String order = conf.get(HTMLPARSEFILTER_ORDER);
        ObjectCache objectCache = ObjectCache.get(conf);
        this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
        if (htmlParseFilters == null) {
          /*
           * If ordered filters are required, prepare array of filters based on
           * property
           */
          String[] orderedFilters = null;
          if (order != null && !order.trim().equals("")) {
            orderedFilters = order.split("\\s+");
          }
            HashMap<String, HtmlParseFilter> filterMap =
              new HashMap<String, HtmlParseFilter>();
            try {
                ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(HtmlParseFilter.X_POINT_ID);
                if (point == null)
                    throw new RuntimeException(HtmlParseFilter.X_POINT_ID + " not found.");
                Extension[] extensions = point.getExtensions();
                for (int i = 0; i < extensions.length; i++) {
                    Extension extension = extensions[i];
                    HtmlParseFilter parseFilter = (HtmlParseFilter) extension.getExtensionInstance();
                    if (!filterMap.containsKey(parseFilter.getClass().getName())) {
                        filterMap.put(parseFilter.getClass().getName(), parseFilter);
                    }
                }
                HtmlParseFilter[] htmlParseFilters = filterMap.values().toArray(new HtmlParseFilter[filterMap.size()]);
                /*
                 * If no ordered filters required, just get the filters in an
                 * indeterminate order
                 */
                if (orderedFilters == null) {
                  objectCache.setObject(HtmlParseFilter.class.getName(), htmlParseFilters);
                }
                /* Otherwise run the filters in the required order */
                else {
                  ArrayList<HtmlParseFilter> filters = new ArrayList<HtmlParseFilter>();
                  for (int i = 0; i < orderedFilters.length; i++) {
                    HtmlParseFilter filter = filterMap
                        .get(orderedFilters[i]);
                    if (filter != null) {
                      filters.add(filter);
                    }
                  }
                  objectCache.setObject(HtmlParseFilter.class.getName(), filters
                      .toArray(new HtmlParseFilter[filters.size()]));
                }
            } catch (PluginRuntimeException e) {
                throw new RuntimeException(e);
            }
            this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
        }
    }                 
View Full Code Here

  private Map<String, String> keyMap = new HashMap<String, String>();
  private Map<String, String> copyMap = new HashMap<String, String>();
  private String uniqueKey = "id";
 
  public static synchronized SolrMappingReader getInstance(Configuration conf) {
    ObjectCache cache = ObjectCache.get(conf);
    SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
    if (instance == null) {
      instance = new SolrMappingReader(conf);
      cache.setObject(SolrMappingReader.class.getName(), instance);
    }
    return instance;
  }
View Full Code Here

    }
  }
 
  public static CollectionManager getCollectionManager(Configuration conf) {
    String key = "collectionmanager";
    ObjectCache objectCache = ObjectCache.get(conf);
    CollectionManager impl = (CollectionManager)objectCache.getObject(key);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Instantiating CollectionManager");
        }
        impl=new CollectionManager(conf);
        objectCache.setObject(key,impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create CollectionManager", e);
      }
    }
    return impl;
View Full Code Here

  private FetchScheduleFactory() {}                   // no public ctor

  /** Return the FetchSchedule implementation. */
  public static FetchSchedule getFetchSchedule(Configuration conf) {
    String clazz = conf.get("db.fetch.schedule.class", DefaultFetchSchedule.class.getName());
    ObjectCache objectCache = ObjectCache.get(conf);
    FetchSchedule impl = (FetchSchedule)objectCache.getObject(clazz);
    if (impl == null) {
      try {
        LOG.info("Using FetchSchedule impl: " + clazz);
        Class implClass = Class.forName(clazz);
        impl = (FetchSchedule)implClass.newInstance();
        impl.setConf(conf);
        objectCache.setObject(clazz, impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create " + clazz, e);
      }
    }
    return impl;
View Full Code Here

  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap
                .get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }                 
View Full Code Here

   * @return The appropriate {@link Protocol} implementation for a given {@link URL}.
   * @throws ProtocolNotFound
   *           when Protocol can not be found for urlString
   */
  public Protocol getProtocol(String urlString) throws ProtocolNotFound {
    ObjectCache objectCache = ObjectCache.get(conf);
    try {
      URL url = new URL(urlString);
      String protocolName = url.getProtocol();
      String cacheId = Protocol.X_POINT_ID + protocolName;
      if (protocolName == null)
        throw new ProtocolNotFound(urlString);

      if (objectCache.getObject(cacheId) != null) {
        return (Protocol) objectCache.getObject(cacheId);
      } else {
        Extension extension = findExtension(protocolName);
        if (extension == null) {
          throw new ProtocolNotFound(protocolName);
        }

        Protocol protocol = (Protocol) extension.getExtensionInstance();

        objectCache.setObject(cacheId, protocol);

        return protocol;
      }

    } catch (MalformedURLException e) {
View Full Code Here

  private SignatureFactory() {}                   // no public ctor

  /** Return the default Signature implementation. */
  public static Signature getSignature(Configuration conf) {
    String clazz = conf.get("db.signature.class", MD5Signature.class.getName());
    ObjectCache objectCache = ObjectCache.get(conf);
    Signature impl = (Signature)objectCache.getObject(clazz);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Using Signature impl: " + clazz);
        }
        Class implClass = Class.forName(clazz);
        impl = (Signature)implClass.newInstance();
        impl.setConf(conf);
        objectCache.setObject(clazz, impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create " + clazz, e);
      }
    }
    return impl;
View Full Code Here

TOP

Related Classes of org.apache.nutch.util.ObjectCache

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.