Package org.apache.nutch.util

Examples of org.apache.nutch.util.ObjectCache


    }
  }
 
  public static CollectionManager getCollectionManager(Configuration conf) {
    String key = "collectionmanager";
    ObjectCache objectCache = ObjectCache.get(conf);
    CollectionManager impl = (CollectionManager)objectCache.getObject(key);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Instantiating CollectionManager");
        }
        impl=new CollectionManager(conf);
        objectCache.setObject(key,impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create CollectionManager",e);
      }
    }
    return impl;
View Full Code Here


   */
  public ResponseWriters(Configuration conf) {

    // get the cache and the cache key
    String cacheKey = ResponseWriter.class.getName();
    ObjectCache objectCache = ObjectCache.get(conf);
    this.responseWriters = (Map<String, ResponseWriter>)objectCache.getObject(cacheKey);

    // if already populated do nothing
    if (this.responseWriters == null) {

      try {

        // get the extension point and all ResponseWriter extensions
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
          ResponseWriter.X_POINT_ID);
        if (point == null) {
          throw new RuntimeException(ResponseWriter.X_POINT_ID + " not found.");
        }

        // populate content type on the ResponseWriter classes, each response
        // writer can handle more than one response type
        Extension[] extensions = point.getExtensions();
        Map<String, ResponseWriter> writers = new HashMap<String, ResponseWriter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          ResponseWriter writer = (ResponseWriter)extension.getExtensionInstance();
          String[] responseTypes = extension.getAttribute("responseType").split(
            ",");
          String contentType = extension.getAttribute("contentType");
          writer.setContentType(contentType);
          for (int k = 0; k < responseTypes.length; k++) {
            writers.put(responseTypes[k], writer);
          }
        }

        // set null object if no writers, otherwise set the writers
        if (writers == null) {
          objectCache.setObject(cacheKey, new HashMap<String, ResponseWriter>());
        }
        else {
          objectCache.setObject(cacheKey, writers);
        }
      }
      catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }

      // set the response writers map
      this.responseWriters = (Map<String, ResponseWriter>)objectCache.getObject(cacheKey);
    }
  }
View Full Code Here

  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filter.addIndexBackendOptions(conf);
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap
                .get(orderedFilters[i]);
            if (filter != null) {
              filter.addIndexBackendOptions(conf);
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }                 
View Full Code Here

          " not found.");
      }
  }

  public static AnalyzerFactory get(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    AnalyzerFactory factory = (AnalyzerFactory) objectCache.getObject(KEY);
    if (factory == null) {
      factory = new AnalyzerFactory(conf);
      objectCache.setObject(KEY, factory);
    }
    return factory;
  }
View Full Code Here

    }
    return analyzer;
  }

  private Extension getExtension(String lang) {
    ObjectCache objectCache = ObjectCache.get(conf);
    if (lang == null) { return null; }
    Extension extension = (Extension) objectCache.getObject(lang);
    if (extension == null) {
      extension = findExtension(lang);
      if (extension != null) {
        objectCache.setObject(lang, extension);
      }
    }
    return extension;
  }
View Full Code Here

   */
  public FieldFilters(Configuration conf) {

    // get the field filter order, the cache, and all field filters
    String order = conf.get(FIELD_FILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.fieldFilters = (FieldFilter[])objectCache.getObject(FieldFilter.class.getName());
   
    if (this.fieldFilters == null) {

      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {

        // get the field filter extension point
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
          FieldFilter.X_POINT_ID);
        if (point == null) {
          throw new RuntimeException(FieldFilter.X_POINT_ID + " not found.");
        }
       
        // get all of the field filter plugins
        Extension[] extensions = point.getExtensions();
        HashMap<String, FieldFilter> filterMap = new HashMap<String, FieldFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          FieldFilter filter = (FieldFilter)extension.getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }

        // order the filters if necessary
        if (orderedFilters == null) {
          objectCache.setObject(FieldFilter.class.getName(),
            filterMap.values().toArray(new FieldFilter[0]));
        }
        else {
          ArrayList<FieldFilter> filters = new ArrayList<FieldFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            FieldFilter filter = filterMap.get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(FieldFilter.class.getName(),
            filters.toArray(new FieldFilter[filters.size()]));
        }
      }
      catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
     
      // set the filters in the cache
      this.fieldFilters = (FieldFilter[])objectCache.getObject(FieldFilter.class.getName());
    }
  }
View Full Code Here

  private Map<String, String> keyMap = new HashMap<String, String>();
  private Map<String, String> copyMap = new HashMap<String, String>();
  private String uniqueKey = "id";
 
  public static synchronized SolrMappingReader getInstance(Configuration conf) {
    ObjectCache cache = ObjectCache.get(conf);
    SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
    if (instance == null) {
      instance = new SolrMappingReader(conf);
      cache.setObject(SolrMappingReader.class.getName(), instance);
    }
    return instance;
  }
View Full Code Here

    if (fields == null) fields = "";
    return Arrays.asList(fields.split("[,\\s]"));
  }

  public QueryFilters(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    this.queryFilters = (QueryFilter[]) objectCache.getObject(QueryFilter.class
        .getName());
    if (this.queryFilters == null) {
      try {
        ExtensionPoint point = PluginRepository.get(conf)
            .getExtensionPoint(QueryFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(QueryFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        FIELD_NAMES = new HashSet<String>();
        RAW_FIELD_NAMES = new HashSet<String>();
        QueryFilter[] filters = new QueryFilter[extensions.length];
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          List<String> fieldNames = parseFieldNames(extension, "fields");
          List<String> rawFieldNames =
            parseFieldNames(extension, "raw-fields");
          if (fieldNames.size() == 0 && rawFieldNames.size() == 0) {
            if (LOG.isWarnEnabled()) {
              LOG.warn("QueryFilter: " + extension.getId()
                     + " names no fields.");
            }
            continue;
          }
          filters[i] = (QueryFilter) extension.getExtensionInstance();
          FIELD_NAMES.addAll(fieldNames);
          FIELD_NAMES.addAll(rawFieldNames);
          objectCache.setObject("FIELD_NAMES", FIELD_NAMES);
          RAW_FIELD_NAMES.addAll(rawFieldNames);
          objectCache.setObject("RAW_FIELD_NAMES", RAW_FIELD_NAMES);
        }
        objectCache.setObject(QueryFilter.class.getName(), filters);
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.queryFilters = (QueryFilter[]) objectCache.getObject(QueryFilter.class
          .getName());
    } else {
      // cache already filled
      FIELD_NAMES = (HashSet<String>) objectCache.getObject("FIELD_NAMES");
      RAW_FIELD_NAMES = (HashSet<String>) objectCache.getObject("RAW_FIELD_NAMES");
    }
  }             
View Full Code Here

    }
  }

  /** Construct using the provided config file. */
  private void init(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    // First, try to retrieve some commonTerms cached in configuration.
    commonTerms = (HashMap<String, HashSet<String>>) objectCache.getObject(KEY);
    if (commonTerms != null) { return; }

    // Otherwise, read the terms.file
    try {
      commonTerms = new HashMap<String, HashSet<String>>();
      Reader reader = conf.getConfResourceAsReader
        (conf.get("analysis.common.terms.file"));
      BufferedReader in = new BufferedReader(reader);
      String line;
      while ((line = in.readLine()) != null) {
        line = line.trim();
        if (line.startsWith("#") || "".equals(line)) // skip comments
          continue;
        TokenStream ts = new NutchDocumentTokenizer(new StringReader(line));
        TermAttribute ta = ts.getAttribute(TermAttribute.class);
        if (!ts.incrementToken()) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line does not contain a field name: " + line);
          }
          continue;
        }
        String field = ta.term();
        if (!ts.incrementToken()) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line contains only a field name, no word: " + line);
          }
          continue;
        }
        String gram = ta.term();
        while (ts.incrementToken()) {
          gram = gram + SEPARATOR + ta.term();
        }
        HashSet<String> table = commonTerms.get(field);
        if (table == null) {
          table = new HashSet<String>();
          commonTerms.put(field, table);
        }
        table.add(gram);
      }
      objectCache.setObject(KEY, commonTerms);
    } catch (IOException e) {
      throw new RuntimeException(e.toString());
    }
  }
View Full Code Here

  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap.get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.nutch.util.ObjectCache

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.