Package org.apache.nutch.util

Examples of org.apache.nutch.util.ObjectCache

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    }
  }
 
  public static CollectionManager getCollectionManager(Configuration conf) {
    String key = "collectionmanager";
    ObjectCache objectCache = ObjectCache.get(conf);
    CollectionManager impl = (CollectionManager)objectCache.getObject(key);
    if (impl == null) {
      try {
        if (LOG.isInfoEnabled()) {
          LOG.info("Instantiating CollectionManager");
        }
        impl=new CollectionManager(conf);
        objectCache.setObject(key,impl);
      } catch (Exception e) {
        throw new RuntimeException("Couldn't create CollectionManager",e);
      }
    }
    return impl;
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
   */
  public ResponseWriters(Configuration conf) {

    // get the cache and the cache key
    String cacheKey = ResponseWriter.class.getName();
    ObjectCache objectCache = ObjectCache.get(conf);
    this.responseWriters = (Map<String, ResponseWriter>)objectCache.getObject(cacheKey);

    // if already populated do nothing
    if (this.responseWriters == null) {

      try {

        // get the extension point and all ResponseWriter extensions
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
          ResponseWriter.X_POINT_ID);
        if (point == null) {
          throw new RuntimeException(ResponseWriter.X_POINT_ID + " not found.");
        }

        // populate content type on the ResponseWriter classes, each response
        // writer can handle more than one response type
        Extension[] extensions = point.getExtensions();
        Map<String, ResponseWriter> writers = new HashMap<String, ResponseWriter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          ResponseWriter writer = (ResponseWriter)extension.getExtensionInstance();
          String[] responseTypes = extension.getAttribute("responseType").split(
            ",");
          String contentType = extension.getAttribute("contentType");
          writer.setContentType(contentType);
          for (int k = 0; k < responseTypes.length; k++) {
            writers.put(responseTypes[k], writer);
          }
        }

        // set null object if no writers, otherwise set the writers
        if (writers == null) {
          objectCache.setObject(cacheKey, new HashMap<String, ResponseWriter>());
        }
        else {
          objectCache.setObject(cacheKey, writers);
        }
      }
      catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }

      // set the response writers map
      this.responseWriters = (Map<String, ResponseWriter>)objectCache.getObject(cacheKey);
    }
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filter.addIndexBackendOptions(conf);
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap
                .get(orderedFilters[i]);
            if (filter != null) {
              filter.addIndexBackendOptions(conf);
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }                 
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
          " not found.");
      }
  }

  public static AnalyzerFactory get(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    AnalyzerFactory factory = (AnalyzerFactory) objectCache.getObject(KEY);
    if (factory == null) {
      factory = new AnalyzerFactory(conf);
      objectCache.setObject(KEY, factory);
    }
    return factory;
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    }
    return analyzer;
  }

  private Extension getExtension(String lang) {
    ObjectCache objectCache = ObjectCache.get(conf);
    if (lang == null) { return null; }
    Extension extension = (Extension) objectCache.getObject(lang);
    if (extension == null) {
      extension = findExtension(lang);
      if (extension != null) {
        objectCache.setObject(lang, extension);
      }
    }
    return extension;
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
   */
  public FieldFilters(Configuration conf) {

    // get the field filter order, the cache, and all field filters
    String order = conf.get(FIELD_FILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.fieldFilters = (FieldFilter[])objectCache.getObject(FieldFilter.class.getName());
   
    if (this.fieldFilters == null) {

      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {

        // get the field filter extension point
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
          FieldFilter.X_POINT_ID);
        if (point == null) {
          throw new RuntimeException(FieldFilter.X_POINT_ID + " not found.");
        }
       
        // get all of the field filter plugins
        Extension[] extensions = point.getExtensions();
        HashMap<String, FieldFilter> filterMap = new HashMap<String, FieldFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          FieldFilter filter = (FieldFilter)extension.getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }

        // order the filters if necessary
        if (orderedFilters == null) {
          objectCache.setObject(FieldFilter.class.getName(),
            filterMap.values().toArray(new FieldFilter[0]));
        }
        else {
          ArrayList<FieldFilter> filters = new ArrayList<FieldFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            FieldFilter filter = filterMap.get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(FieldFilter.class.getName(),
            filters.toArray(new FieldFilter[filters.size()]));
        }
      }
      catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
     
      // set the filters in the cache
      this.fieldFilters = (FieldFilter[])objectCache.getObject(FieldFilter.class.getName());
    }
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
  private Map<String, String> keyMap = new HashMap<String, String>();
  private Map<String, String> copyMap = new HashMap<String, String>();
  private String uniqueKey = "id";
 
  public static synchronized SolrMappingReader getInstance(Configuration conf) {
    ObjectCache cache = ObjectCache.get(conf);
    SolrMappingReader instance = (SolrMappingReader)cache.getObject(SolrMappingReader.class.getName());
    if (instance == null) {
      instance = new SolrMappingReader(conf);
      cache.setObject(SolrMappingReader.class.getName(), instance);
    }
    return instance;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    if (fields == null) fields = "";
    return Arrays.asList(fields.split("[,\\s]"));
  }

  public QueryFilters(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    this.queryFilters = (QueryFilter[]) objectCache.getObject(QueryFilter.class
        .getName());
    if (this.queryFilters == null) {
      try {
        ExtensionPoint point = PluginRepository.get(conf)
            .getExtensionPoint(QueryFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(QueryFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        FIELD_NAMES = new HashSet<String>();
        RAW_FIELD_NAMES = new HashSet<String>();
        QueryFilter[] filters = new QueryFilter[extensions.length];
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          List<String> fieldNames = parseFieldNames(extension, "fields");
          List<String> rawFieldNames =
            parseFieldNames(extension, "raw-fields");
          if (fieldNames.size() == 0 && rawFieldNames.size() == 0) {
            if (LOG.isWarnEnabled()) {
              LOG.warn("QueryFilter: " + extension.getId()
                     + " names no fields.");
            }
            continue;
          }
          filters[i] = (QueryFilter) extension.getExtensionInstance();
          FIELD_NAMES.addAll(fieldNames);
          FIELD_NAMES.addAll(rawFieldNames);
          objectCache.setObject("FIELD_NAMES", FIELD_NAMES);
          RAW_FIELD_NAMES.addAll(rawFieldNames);
          objectCache.setObject("RAW_FIELD_NAMES", RAW_FIELD_NAMES);
        }
        objectCache.setObject(QueryFilter.class.getName(), filters);
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.queryFilters = (QueryFilter[]) objectCache.getObject(QueryFilter.class
          .getName());
    } else {
      // cache already filled
      FIELD_NAMES = (HashSet<String>) objectCache.getObject("FIELD_NAMES");
      RAW_FIELD_NAMES = (HashSet<String>) objectCache.getObject("RAW_FIELD_NAMES");
    }
  }             
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
    }
  }

  /** Construct using the provided config file. */
  private void init(Configuration conf) {
    ObjectCache objectCache = ObjectCache.get(conf);
    // First, try to retrieve some commonTerms cached in configuration.
    commonTerms = (HashMap<String, HashSet<String>>) objectCache.getObject(KEY);
    if (commonTerms != null) { return; }

    // Otherwise, read the terms.file
    try {
      commonTerms = new HashMap<String, HashSet<String>>();
      Reader reader = conf.getConfResourceAsReader
        (conf.get("analysis.common.terms.file"));
      BufferedReader in = new BufferedReader(reader);
      String line;
      while ((line = in.readLine()) != null) {
        line = line.trim();
        if (line.startsWith("#") || "".equals(line)) // skip comments
          continue;
        TokenStream ts = new NutchDocumentTokenizer(new StringReader(line));
        TermAttribute ta = ts.getAttribute(TermAttribute.class);
        if (!ts.incrementToken()) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line does not contain a field name: " + line);
          }
          continue;
        }
        String field = ta.term();
        if (!ts.incrementToken()) {
          if (LOG.isWarnEnabled()) {
            LOG.warn("Line contains only a field name, no word: " + line);
          }
          continue;
        }
        String gram = ta.term();
        while (ts.incrementToken()) {
          gram = gram + SEPARATOR + ta.term();
        }
        HashSet<String> table = commonTerms.get(field);
        if (table == null) {
          table = new HashSet<String>();
          commonTerms.put(field, table);
        }
        table.add(gram);
      }
      objectCache.setObject(KEY, commonTerms);
    } catch (IOException e) {
      throw new RuntimeException(e.toString());
    }
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
  private IndexingFilter[] indexingFilters;

  public IndexingFilters(Configuration conf) {
    /* Get indexingfilter.order property */
    String order = conf.get(INDEXINGFILTER_ORDER);
    ObjectCache objectCache = ObjectCache.get(conf);
    this.indexingFilters = (IndexingFilter[]) objectCache
        .getObject(IndexingFilter.class.getName());
    if (this.indexingFilters == null) {
      /*
       * If ordered filters are required, prepare array of filters based on
       * property
       */
      String[] orderedFilters = null;
      if (order != null && !order.trim().equals("")) {
        orderedFilters = order.split("\\s+");
      }
      try {
        ExtensionPoint point = PluginRepository.get(conf).getExtensionPoint(
            IndexingFilter.X_POINT_ID);
        if (point == null)
          throw new RuntimeException(IndexingFilter.X_POINT_ID + " not found.");
        Extension[] extensions = point.getExtensions();
        HashMap<String, IndexingFilter> filterMap =
          new HashMap<String, IndexingFilter>();
        for (int i = 0; i < extensions.length; i++) {
          Extension extension = extensions[i];
          IndexingFilter filter = (IndexingFilter) extension
              .getExtensionInstance();
          LOG.info("Adding " + filter.getClass().getName());
          if (!filterMap.containsKey(filter.getClass().getName())) {
            filterMap.put(filter.getClass().getName(), filter);
          }
        }
        /*
         * If no ordered filters required, just get the filters in an
         * indeterminate order
         */
        if (orderedFilters == null) {
          objectCache.setObject(IndexingFilter.class.getName(),
              filterMap.values().toArray(
                  new IndexingFilter[0]));
          /* Otherwise run the filters in the required order */
        } else {
          ArrayList<IndexingFilter> filters = new ArrayList<IndexingFilter>();
          for (int i = 0; i < orderedFilters.length; i++) {
            IndexingFilter filter = filterMap.get(orderedFilters[i]);
            if (filter != null) {
              filters.add(filter);
            }
          }
          objectCache.setObject(IndexingFilter.class.getName(), filters
              .toArray(new IndexingFilter[filters.size()]));
        }
      } catch (PluginRuntimeException e) {
        throw new RuntimeException(e);
      }
      this.indexingFilters = (IndexingFilter[]) objectCache
          .getObject(IndexingFilter.class.getName());
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.util.ObjectCache

  • org.apache.nutch.analysis.AnalyzerFactory
  • org.apache.nutch.analysis.CommonGrams
  • org.apache.nutch.collection.CollectionManager
  • org.apache.nutch.crawl.FetchScheduleFactory
  • org.apache.nutch.crawl.SignatureFactory
  • org.apache.nutch.indexer.field.FieldFilters
  • org.apache.nutch.indexer.IndexingFilters
  • org.apache.nutch.indexer.IndexWriters
  • org.apache.nutch.indexer.solr.SolrMappingReader
  • org.apache.nutch.indexwriter.solr.SolrMappingReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.