Examples of ObjectCache


Examples of org.apache.nutch.util.ObjectCache

  public Parser getParserById(String id) throws ParserNotFound {

    Extension[] extensions = this.extensionPoint.getExtensions();
    Extension parserExt = null;

    ObjectCache objectCache = ObjectCache.get(conf);
   
    if (id != null) {
      parserExt = getExtension(extensions, id);
    }
    if (parserExt == null) {
      parserExt = getExtensionFromAlias(extensions, id);
    }

    if (parserExt == null) {
      throw new ParserNotFound("No Parser Found for id [" + id + "]");
    }
   
    // first check the cache          
    if (objectCache.getObject(parserExt.getId()) != null) {
      return (Parser) objectCache.getObject(parserExt.getId());

    // if not found in cache, instantiate the Parser   
    } else {
      try {
        Parser p = (Parser) parserExt.getExtensionInstance();
        objectCache.setObject(parserExt.getId(), p);
        return p;
      } catch (PluginRuntimeException e) {
        if (LOG.isWarnEnabled()) {
          LOG.warn("Canno initialize parser " +
                   parserExt.getDescriptor().getPluginId() +
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

   * @return a list of extensions to be used for this contentType.
   *         If none, returns <code>null</code>.
   */
  protected List<Extension> getExtensions(String contentType) {
   
    ObjectCache objectCache = ObjectCache.get(conf);
    // First of all, tries to clean the content-type
    String type = null;
    type = MimeUtil.cleanMimeType(contentType);


    List<Extension> extensions = (List<Extension>) objectCache.getObject(type);

    // Just compare the reference:
    // if this is the empty list, we know we will find no extension.
    if (extensions == EMPTY_EXTENSION_LIST) {
      return null;
    }
   
    if (extensions == null) {
      extensions = findExtensions(type);
      if (extensions != null) {
        objectCache.setObject(type, extensions);
      } else {
        // Put the empty extension list into cache
        // to remember we don't know any related extension.
        objectCache.setObject(type, EMPTY_EXTENSION_LIST);
      }
    }
    return extensions;
  }
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

public class HtmlParseFilters {

  private HtmlParseFilter[] htmlParseFilters;

  public HtmlParseFilters(Configuration conf) {
        ObjectCache objectCache = ObjectCache.get(conf);
        this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
        if (htmlParseFilters == null) {
            HashMap<String, HtmlParseFilter> filters =
              new HashMap<String, HtmlParseFilter>();
            try {
                ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(HtmlParseFilter.X_POINT_ID);
                if (point == null)
                    throw new RuntimeException(HtmlParseFilter.X_POINT_ID + " not found.");
                Extension[] extensions = point.getExtensions();
                for (int i = 0; i < extensions.length; i++) {
                    Extension extension = extensions[i];
                    HtmlParseFilter parseFilter = (HtmlParseFilter) extension.getExtensionInstance();
                    if (!filters.containsKey(parseFilter.getClass().getName())) {
                        filters.put(parseFilter.getClass().getName(), parseFilter);
                    }
                }
                HtmlParseFilter[] htmlParseFilters = filters.values().toArray(new HtmlParseFilter[filters.size()]);
                objectCache.setObject(HtmlParseFilter.class.getName(), htmlParseFilters);
            } catch (PluginRuntimeException e) {
                throw new RuntimeException(e);
            }
            this.htmlParseFilters = (HtmlParseFilter[]) objectCache.getObject(HtmlParseFilter.class.getName());
        }
    }                 
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

    }
  }

  /** Construct using the provided config file. */
  private void init(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    // First, try to retrieve some commonTerms cached in configuration.
    commonTerms = (HashMap<String, HashSet<String>>) objectCache.getObject(KEY);
    if (commonTerms != null) { return; }

    // Otherwise, read the terms.file
    try {
      commonTerms = new HashMap<String, HashSet<String>>();
      Reader reader = conf.getConfResourceAsReader
        (conf.get("analysis.common.terms.file"));
      BufferedReader in = new BufferedReader(reader);
      String line;
      while ((line = in.readLine()) != null) {
        line = line.trim();
        if (line.startsWith("#") || "".equals(line)) // skip comments
          continue;
        TokenStream ts = new NutchDocumentTokenizer(new StringReader(line));
        Token token = ts.next();
        if (token == null) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line does not contain a field name: " + line);
          }
          continue;
        }
        String field = token.termText();
        token = ts.next();
        if (token == null) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line contains only a field name, no word: " + line);
          }
          continue;
        }
        String gram = token.termText();
        while ((token = ts.next()) != null) {
          gram = gram + SEPARATOR + token.termText();
        }
        HashSet<String> table = commonTerms.get(field);
        if (table == null) {
          table = new HashSet<String>();
          commonTerms.put(field, table);
        }
        table.add(gram);
      }
      objectCache.setObject(KEY, commonTerms);
    } catch (IOException e) {
      throw new RuntimeException(e.toString());
    }
  }
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

   * @return array of plugin instances
   */
  public synchronized Object[] getOrderedPlugins(Class<?> clazz, String xPointId,
      String orderProperty) {
    Object[] filters;
    ObjectCache objectCache = ObjectCache.get(conf);
    filters = (Object[]) objectCache.getObject(clazz.getName());

    if (filters == null) {
      String order = conf.get(orderProperty);
      List<String> orderOfFilters = new ArrayList<String>();
      boolean userDefinedOrder = false;
      if (order != null && !order.trim().isEmpty()) {
        orderOfFilters = Arrays.asList(order.trim().split("\\s+"));
        userDefinedOrder = true;
      }

      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            xPointId);
        if (point == null)
          throw new RuntimeException(xPointId + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, Object> filterMap = new HashMap<String, Object>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          Object filter = extension.getExtensionInstance();
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
            if (!userDefinedOrder)
              orderOfFilters.add(filter.getClass().getName());
          }
        }
        List<Object> sorted = new ArrayList<Object>();
        for (String orderedFilter : orderOfFilters) {
          Object f = filterMap.get(orderedFilter);
          if (f == null) {
            LOG.error(clazz.getSimpleName() + " : " + orderedFilter
                + " declared in configuration property " + orderProperty
                + " but not found in an active plugin - ignoring.");
            continue;
          }
          sorted.add(f);
        }
        Object[] filter = (Object[]) Array.newInstance(clazz, sorted.size());
        for (int i = 0; i < sorted.size(); i++) {
          filter[i] = sorted.get(i);
          if (LOG.isTraceEnabled()) {
            LOG.trace(clazz.getSimpleName() + " : filters[" + i + "] = "
                + filter[i].getClass());
          }
        }
        objectCache.setObject(clazz.getName(), filter);
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }

      filters = (Object[]) objectCache.getObject(clazz.getName());
    }
    return filters;
  }
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

  private SignatureFactory() {}                   // no public ctor

  /** Return the default Signature implementation. */
  public static Signature getSignature(Configuration conf) {
    String clazz = conf.get("db.signature.class", MD5Signature.class.getName());
    ObjectCache objectCache = ObjectCache.get(conf);
    Signature impl = (Signature)objectCache.getObject(clazz);
    if (impl == null) {
      try {
        LOG.info("Using Signature impl: " + clazz);
        Class<?> implClass = Class.forName(clazz);
        impl = (Signature)implClass.newInstance();
        impl.setConf(conf);
        objectCache.setObject(clazz, impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create " + clazz, e);
      }
    }
    return impl;
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap.get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }
View Full Code Here

Examples of org.apache.nutch.util.ObjectCache

  private FetchScheduleFactory() {}                   // no public ctor

  /** Return the FetchSchedule implementation. */
  public static FetchSchedule getFetchSchedule(Configuration conf) {
    String clazz = conf.get("db.fetch.schedule.class", DefaultFetchSchedule.class.getName());
    ObjectCache objectCache = ObjectCache.get(conf);
    FetchSchedule impl = (FetchSchedule)objectCache.getObject(clazz);
    if (impl == null) {
      try {
        LOG.info("Using FetchSchedule impl: " + clazz);
        Class<?> implClass = Class.forName(clazz);
        impl = (FetchSchedule)implClass.newInstance();
        impl.setConf(conf);
        objectCache.setObject(clazz, impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create " + clazz, e);
      }
    }
    return impl;
View Full Code Here

Examples of org.apache.ojb.broker.cache.ObjectCache

        ConnectionManagerIF connMan = _pb.serviceConnectionManager();
        boolean saveBatchMode = connMan.isBatchMode();
        Swizzling swizzlingStrategy = _tx.getKit().getSwizzlingStrategy();
        LockManager lockManager = LockManager.getInstance();
        Identity[] lockOrder = (Identity[]) _order.toArray(new Identity[_order.size()]);
        ObjectCache cache = _pb.serviceObjectCache();
        boolean isInsertVerified = _tx.getKit().isInsertVerified();
        ArrayList changedCollections = new ArrayList();

        // sort objects in the order of oid.hashCode to avoid deadlocks
        Arrays.sort(lockOrder, new Comparator()
        {
            public int compare(Object o1, Object o2)
            {
                return o1.hashCode() - o2.hashCode();
            }

            public boolean equals(Object obj)
            {
                return false;
            }
        });

        try {
            // mark dirty objects and lock them for write
            // also handle dependent objects and if there were inserted once,
            // repeat this process for their dependants ("cascade create")
            ArrayList newObjects = new ArrayList();
            int countNewObjects;
            do
            {
                newObjects.clear();
                countNewObjects = 0;
                for (int i = 0; i < lockOrder.length; i++)
                {
                    Identity oid = lockOrder[i];
                    ContextEntry entry = (ContextEntry) _objects.get(oid);
                    State state = entry.state;

                    if (entry.userObject == null) // invalidated
                    {
                        continue;
                    }

                    if (entry.handler == null) // materialized
                    {
                        if (!state.isDeleted())
                        {
                            Object[][] origFields = (Object[][]) _checkpointed.get(oid);
                            Object[][] newFields = getFields(entry.userObject, true, !isCommit);

                            if (origFields == null)
                            {
                                entry.needsCacheSwizzle = true;
                                newObjects.addAll(
                                        handleDependentReferences(oid, entry.userObject,
                                        null, newFields[0], newFields[2]));
                                newObjects.addAll(
                                        handleDependentCollections(oid, entry.userObject,
                                        null, newFields[1], newFields[3]));
                            }
                            else
                            {
                                if (isModified(origFields[0], newFields[0]))
                                {
                                    entry.state = state.markDirty();
                                    entry.needsCacheSwizzle = true;
                                    lockManager.ensureLock(oid, _tx, LockType.WRITE_LOCK, _pb);
                                    newObjects.addAll(
                                            handleDependentReferences(oid, entry.userObject,
                                            origFields[0], newFields[0], newFields[2]));
                                }

                                if (isModified(origFields[1], newFields[1]))
                                {
                                    // there are modified collections,
                                    // so we need to lock the object and to swizzle it to cache
                                    entry.needsCacheSwizzle = true;
                                    lockManager.ensureLock(oid, _tx, LockType.WRITE_LOCK, _pb);
                                    newObjects.addAll(
                                            handleDependentCollections(oid, entry.userObject,
                                            origFields[1], newFields[1], newFields[3]));
                                    changedCollections.add(oid);
                                }
                            }
                        }
                    }
                }
                countNewObjects = newObjects.size();
                if (countNewObjects > 0)
                {
                    // new objects are not locked, so we don't need to ensure the order
                    lockOrder = (Identity[]) newObjects.toArray(
                            new Identity[countNewObjects]);
                }
            }
            while (countNewObjects > 0);

            // Swizzle the context objects and the cache objects
            for (Iterator it = _order.iterator(); it.hasNext(); )
            {
                Identity oid = (Identity) it.next();
                ContextEntry entry = (ContextEntry) _objects.get(oid);

                if (entry.needsCacheSwizzle)
                {
                    entry.userObject = swizzlingStrategy.getRealTarget(entry.userObject);
                    entry.cacheObject = swizzlingStrategy.swizzle(
                    // we create the special ObjectCache implememntation
                            // that returns cacheObject, not userObject
                            entry.userObject, entry.cacheObject, _pb, new ObjectCache()
                            {
                                public Object lookup(Identity anOid)
                                {
                                    ContextEntry ent = (ContextEntry) _objects.get(anOid);
                                    return (ent == null ? null : ent.cacheObject);
                                }

                                public boolean contains(Identity oid)
                                {
                                    return lookup(oid) != null;
                                }

                                public void cache(Identity anOid, Object obj)
                                {
                                    // do nothing
                                }

                                public boolean cacheIfNew(Identity oid, Object obj)
                                {
                                    return false;
                                }

                                public void clear()
                                {
                                    // do nothing
                                }

                                public void remove(Identity anOid)
                                {
                                    // do nothing
                                }
                            });
                }
            }

            // Cascade delete for dependent objects
            int countCascadeDeleted;
            do
            {
                countCascadeDeleted = 0;
                // Use intermediate new ArrayList(_order) because _order
                // may be changed during cascade delete
                for (Iterator it = (new ArrayList(_order)).iterator(); it.hasNext(); )
                {
                    Identity oid = (Identity) it.next();
                    ContextEntry entry = (ContextEntry) _objects.get(oid);

                    if (entry.state.isDeleted())
                    {
                        countCascadeDeleted += doCascadeDelete(oid, entry.userObject);
                    }
                }
            }
            while (countCascadeDeleted > 0);

            // perform database operations
            connMan.setBatchMode(true);
            try
            {
                for (Iterator it = _order.iterator(); it.hasNext(); )
                {
                    Identity oid = (Identity) it.next();
                    ContextEntry entry = (ContextEntry) _objects.get(oid);
                    State state = entry.state;

                    if (!state.needsInsert() && !state.needsUpdate()
                            && !state.needsDelete())
                    {
                        if (changedCollections.contains(oid)) {
                            _pb.store(entry.cacheObject, state);
                        }
                        continue;
                    }

                    if (state.needsInsert())
                    {
                        if (isInsertVerified)
                        {
                            // PB verifies object existence by default
                            _pb.store(entry.cacheObject);
                        }
                        else
                        {
                            // PB migth already created the object by auto-update
                            if (cache.lookup(oid) == null) {
                                _pb.store(entry.cacheObject, state);
                            }
                        }

                    }
View Full Code Here

Examples of org.apache.ojb.broker.cache.ObjectCache

        Class topLevelClass = getBroker().getTopLevelClass(ord.getItemClass());
        Object[] fkValues;
        Object owner;
        Identity id;
        PersistenceBroker pb = getBroker();
        ObjectCache cache = pb.serviceObjectCache();

        while (iter.hasNext())
        {
            owner = iter.next();
            fkValues = ord.getForeignKeyValues(owner,cld);
            if (isNull(fkValues))
            {
                continue;
            }
            id = pb.serviceIdentity().buildIdentity(null, topLevelClass, fkValues);
            if (cache.lookup(id) != null)
            {
                children.add(pb.getObjectByIdentity(id));
                continue;
            }
            idsSubset.add(id);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.