* Licensed to the Apache Software Foundation (ASF) under one or more

* contributor license agreements.  See the NOTICE file distributed with

* this work for additional information regarding copyright ownership.

* The ASF licenses this file to You under the Apache License, Version 2.0

* (the "License"); you may not use this file except in compliance with

* the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This class provides an interface to the OpenCalais service for Named Entity Recognition.

* It uses the OpenCalais REST service with the 'paramsXML' structures for passing

* parameters {@link http://www.opencalais.com/documentation/calais-web-service-api/api-invocation/rest-using-paramsxml)}.

*

* @author <a href="mailto:kasper@dfki.de">Walter Kasper</a>

*/

@Properties(value={

    @Property(name=EnhancementEngine.PROPERTY_NAME,value="opencalais")

})

        extends AbstractEnhancementEngine<RuntimeException,RuntimeException>

        implements EnhancementEngine, ServiceProperties {

    private static Logger log = LoggerFactory.getLogger(OpenCalaisEngine.class);

    /**

     * This contains the directly supported MIME types of this enhancement engine.

     */

    protected static final Set<String> SUPPORTED_MIMETYPES =

            Collections.unmodifiableSet(new HashSet<String>(

                    Arrays.asList("text/plain", "text/html")));

    /**

     * This contains a list of languages supported by OpenCalais.

     * If the metadata don't contain a value for the language as the value of the {@link Property.DC_LANG property}

     * it is left to the grace of the OpenCalais whether it accepts the text.

     * OpenCalais uses its own language identifcation anyway.

     */

    protected static final Set<String> SUPPORTED_LANGUAGES =

            Collections.unmodifiableSet(new HashSet<String>(

                    Arrays.asList("en", "fr", "es")));

    /**

     * The default value for the Execution of this Engine. Currently set to

     * {@link ServiceProperties#ORDERING_EXTRACTION_ENHANCEMENT} + 10. It should run after Metaxa and LangId.

     */

    public static final Integer defaultOrder = ServiceProperties.ORDERING_EXTRACTION_ENHANCEMENT + 10;

    @Property

    public static final String LICENSE_KEY = "org.apache.stanbol.enhancer.engines.opencalais.license";

    @Property(value = "http://api.opencalais.com/enlighten/rest/")

    public static final String CALAIS_URL_KEY = "org.apache.stanbol.enhancer.engines.opencalais.url";

   

    @Property

    public static final String CALAIS_TYPE_MAP_KEY = "org.apache.stanbol.enhancer.engines.opencalais.typeMap";

   

    @Property(value="true")

    public static final String CALAIS_NER_ONLY_MODE_KEY = "org.apache.stanbol.enhancer.engines.opencalais.NERonly";

    /**

     * the URL for the Calais REST Service

     */

    private String calaisUrl = "http://api.opencalais.com/enlighten/rest/";

    /**

     * the license key from OpenCalais for using the service

     */

    private String licenseKey = null;

   

    private String calaisTypeMapFile = null;

    /**

     * specify whether only the NER results from OpenCalais should be used. Entity references from OpenCalais will be omitted. This mode is intended to be used with another entity tagging engine.

     */

    private boolean onlyNERMode;

    @Reference

    TcManager tcManager;

    /**

     * Only activate this engine in online mode

     */

    @SuppressWarnings("unused")

    @Reference

    private OnlineMode onlineMode;

    BundleContext bundleContext;

    /**

     * a map for mapping Calais classes to other classes (e.g. from dbpedia)

     */

    private Map<UriRef,UriRef> calaisTypeMap;

   

    /**

     * the default file containing type mappings. Key and value are separated by the regular expression ' ?= ?'.

     */

    private static final String CALAIS_TYPE_MAP_DEFAULT ="calaisTypeMap.txt";

    public String getLicenseKey() {

        return licenseKey;

    }

    public void setLicenseKey(String licenseKey) throws ConfigurationException {

        if(licenseKey == null || licenseKey.isEmpty()){

            throw new ConfigurationException(LICENSE_KEY, String.format(

                "%s : please configure a OpenCalais License key to use this engine (e.g. by" +

                "using the 'Configuration' tab of the Apache Felix Web Console).",

                getClass().getSimpleName()));

        }

        this.licenseKey = licenseKey;

    }

    public String getCalaisUrl() {

        return calaisUrl;

    }

    public void setCalaisUrl(String calaisUrl) throws ConfigurationException {

        if(calaisUrl == null || calaisUrl.isEmpty()){

            throw new ConfigurationException(CALAIS_URL_KEY, String.format(

                "%s : please configure the URL of the OpenCalais Webservice (e.g. by" +

                "using the 'Configuration' tab of the Apache Felix Web Console).",

                getClass().getSimpleName()));

        }

        this.calaisUrl = calaisUrl;

    }

    public Map<UriRef,UriRef> getCalaisTypeMap() {

      return calaisTypeMap;

    }

    public void setCalaisTypeMap(Map<UriRef,UriRef> calaisTypeMap) {

      this.calaisTypeMap = calaisTypeMap;

    }

    public Map<String, Object> getServiceProperties() {

        // TODO Auto-generated method stub

        return Collections.unmodifiableMap(Collections.singletonMap(

                ENHANCEMENT_ENGINE_ORDERING,

                (Object) defaultOrder));

    }

   

    protected void loadTypeMap(String resource) {

      InputStream in = null;

      BufferedReader reader = null;

      try {

      if (resource != null && resource.trim().length()>0) {

        in = new FileInputStream(resource);

      }

      else {

        in = getClass().getClassLoader().getResourceAsStream(CALAIS_TYPE_MAP_DEFAULT);

      }

      reader = new BufferedReader(new InputStreamReader(in));

      String line;

        while ((line = (reader.readLine())) != null) {

          if (line.startsWith("#"))

            continue;

          String[] entry = line.split("\\s*=\\s*");

          if (entry.length == 2) {

            calaisTypeMap.put(new UriRef(entry[0]), new UriRef(entry[1]));

          }

        }

        reader.close();

        in.close();

      }

      catch (IOException e) {

        log.error("Error in reading type map file: {}", e.getMessage());

      }

    }

    public int canEnhance(ContentItem ci) throws EngineException {

        if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null){

            String language = EnhancementEngineHelper.getLanguage(ci);

            if (language != null && !SUPPORTED_LANGUAGES.contains(language)) {

                log.info("OpenCalais can not process ContentItem {} because "

                    + "language {} is not supported (supported: {})",

                    new Object[]{ci.getUri(),language,SUPPORTED_LANGUAGES});

                return CANNOT_ENHANCE;

            }

            return ENHANCE_ASYNC; //OpenCalais now support async processing!

        }

        return CANNOT_ENHANCE;

    }

    public void computeEnhancements(ContentItem ci) throws EngineException {

        Entry<UriRef,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES);

        if(contentPart == null){

            throw new IllegalStateException("No ContentPart with an supported Mimetype '"

                    + SUPPORTED_MIMETYPES+"' found for ContentItem "+ci.getUri()

                    + ": This is also checked in the canEnhance method! -> This "

                    + "indicated an Bug in the implementation of the "

                    + "EnhancementJobManager!");

        }

        String text;

        try {

            text = ContentItemHelper.getText(contentPart.getValue());

        } catch (IOException e) {

            throw new InvalidContentException(this, ci, e);

        }

        MGraph calaisModel = getCalaisAnalysis(text, contentPart.getValue().getMimeType());

        if (calaisModel != null) {

            //Acquire a write lock on the ContentItem when adding the enhancements

            ci.getLock().writeLock().lock();

            try {

                createEnhancements(queryModel(calaisModel), ci);

                if (log.isDebugEnabled()) {

                    Serializer serializer = Serializer.getInstance();

                    ByteArrayOutputStream debugStream = new ByteArrayOutputStream();

                    serializer.serialize(debugStream, ci.getMetadata(), "application/rdf+xml");

                    try {

                        log.debug("Calais Enhancements:\n{}",debugStream.toString("UTF-8"));

                    } catch (UnsupportedEncodingException e) {

                        e.printStackTrace();

                    }

                }

            } finally {

                ci.getLock().writeLock().unlock();

            }

        }

    }

    /**

     * This generates enhancement structures for the entities from OpenCalais

     * and adds them to the content item's metadata.

     * For each entity a TextAnnotation and an EntityAnnotation are created.

     * An EntityAnnotation can relate to several TextAnnotations.

     *

     * @param occs a Collection of entity information

     * @param ci the content item

     */

    public void createEnhancements(Collection<CalaisEntityOccurrence> occs, ContentItem ci) {

        LiteralFactory literalFactory = LiteralFactory.getInstance();

        final Language language; // used for plain literals representing parts fo the content

        String langString = EnhancementEngineHelper.getLanguage(ci);

        if(langString != null && !langString.isEmpty()){

            language = new Language(langString);

        } else {

            language = null;

        }

        //TODO create TextEnhancement (form, start, end, type?) and EntityAnnotation (id, name, type)

        HashMap<Resource, UriRef> entityAnnotationMap = new HashMap<Resource, UriRef>();

        for (CalaisEntityOccurrence occ : occs) {

            UriRef textAnnotation = EnhancementEngineHelper.createTextEnhancement(

                    ci, this);

            MGraph model = ci.getMetadata();

            model.add(new TripleImpl(textAnnotation, DC_TYPE, occ.type));

            // for autotagger use the name instead of the matched term (that might be a pronoun!)

            if (onlyNERMode) {

                model.add(new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT,new PlainLiteralImpl(occ.name,language)));

            }

            else {

                model.add(new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT, new PlainLiteralImpl(occ.exact,language)));

            }

            model.add(new TripleImpl(textAnnotation, ENHANCER_START, literalFactory.createTypedLiteral(occ.offset)));

            model.add(new TripleImpl(textAnnotation, ENHANCER_END, literalFactory.createTypedLiteral(occ.offset + occ.length)));

            model.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT, new PlainLiteralImpl(occ.context,language)));

            //use the relevance as confidence

            if(occ.relevance != null && Double.valueOf(0).compareTo(occ.relevance) <= 0 ){

                //we do not know if the relevance is available (may be NULL)

                //or the relevance feature is activated (may be -1)

                model.add(new TripleImpl(textAnnotation, ENHANCER_CONFIDENCE, literalFactory.createTypedLiteral(occ.relevance)));

            }

            //create EntityAnnotation only once but add a reference to the textAnnotation

            if (entityAnnotationMap.containsKey(occ.id)) {

                model.add(new TripleImpl(entityAnnotationMap.get(occ.id), DC_RELATION, textAnnotation));

            } else {

                if (onlyNERMode) {

              // don't create Calais specific entity annotations; let the autotagger do its's own

              // but add a pointer to the first text annotation with that name

                entityAnnotationMap.put(occ.id,textAnnotation);

                }

                else {

//                entityAnnotationMap.put(occ.id, entityAnnotation);

//                model.add(new TripleImpl(entityAnnotation, DC_RELATION, textAnnotation));

//                model.add(new TripleImpl(entityAnnotation, ENHANCER_ENTITY_LABEL, occ.name));

//                model.add(new TripleImpl(entityAnnotation, ENHANCER_ENTITY_TYPE, occ.type));

//                model.add(new TripleImpl(entityAnnotation, ENHANCER_ENTITY_REFERENCE, occ.id));

                }

            }

        }

    }

    /**

     * Retrieves the annotations from OpenCalais as RDF/XML. From that an MGraph is created.

     *

     * @param text the text to send to OpenCalais

     *

     * @return an MGraph with all annotations

     *

     * @throws EngineException

     */

    public MGraph getCalaisAnalysis(String text, String mimeType) throws EngineException {

        if (mimeType.equals("text/plain")) {

            mimeType = "text/raw";

        }

        String calaisParams = "<c:params xmlns:c=\"http://s.opencalais.com/1/pred/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">" +

                "<c:processingDirectives c:contentType=\"" + mimeType + "\" " +

                //    "c:enableMetadataType=\"GenericRelations,SocialTags\" "+

                //    "c:enableMetadataType=\"GenericRelations\" "+

                "c:outputFormat=\"rdf/xml\" " +

                //NOTE (rw, 2012-05-29) changed to true while working on STANBOL-630

                "c:calculateRelevanceScore=\"true\" " +

                "c:omitOutputtingOriginalText=\"true\"" +

                ">" +

                "</c:processingDirectives>" +

                "</c:params>";

        MGraph model = null;

        try {

            StringBuilder postParams = new StringBuilder();

            postParams

                    .append("licenseID=")

                    .append(URLEncoder.encode(getLicenseKey(), "UTF-8"))

                    .append("&content=")

                    .append(URLEncoder.encode(text, "UTF-8"))

                    .append("&paramsXML=")

                    .append(URLEncoder.encode(calaisParams, "UTF-8"));

            // get annotations from Calais

            log.info("Calais request sent");

            String calaisResult =

                    doPostRequest(

                            this.getCalaisUrl(), null, postParams.toString(),

                            "application/x-www-form-urlencoded", "UTF-8");

            log.info("Calais response received: {}",calaisResult.length());

            log.info("Calais response:\n {}",calaisResult);

            log.debug("Calais data:\n{}", calaisResult);

            // build model from Calais result

            InputStream in = new ByteArrayInputStream(calaisResult.getBytes("utf-8"));

            model = readModel(in, "application/rdf+xml");

        } catch (UnsupportedEncodingException e) {

            throw new EngineException(e.getMessage(), e);

        } catch (IOException e) {

            throw new EngineException(e.getMessage(), e);

        }

        return model;

    }

    /**

     * Parses an InputStream of RDF data and produces an MGraph from them

     *

     * @param in The InputStream of RDF data

     * @param format the format of the RDF data

     *

     * @return the resulting MGraph or null if the RDF serialization format is not supported by the parser

     */

    public MGraph readModel(InputStream in, String format) {

        Parser parser = Parser.getInstance();

        if (parser.getSupportedFormats().contains(format)) {

            Graph graph = parser.parse(in, format);

            MGraph model = new SimpleMGraph(graph);

            return model;

        } else {

            log.warn("Unsupported RDF format: {}\nSupported RDF formats: {}",

                    format, parser.getSupportedFormats());

        }

        return null;

    }

    /**

     * Extracts the relevant entity information from the Calais RDF data.

     * The entities and the relted information is extracted by a Sparql query.

     *

     * @param model the MGraph representing the Calais data

     *

     * @return a Collection of entity information

     * @throws EngineException on a {@link ParseException} while processing the

     * Sparql query.

     */

    public Collection<CalaisEntityOccurrence> queryModel(MGraph model) throws EngineException {

        //TODO extract also Geo info (latitude/longitude)?

        String query =

                "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> " +

                        "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> " +

                        "PREFIX p: <http://s.opencalais.com/1/pred/> " +

                        "PREFIX t: <http://s.opencalais.com/1/type/em/e/> " +

                        "SELECT DISTINCT ?id ?did ?name ?type ?dtype ?offset ?length ?exact ?context ?score WHERE { " +

                        "?id p:name ?name ." +

                        "?id rdf:type ?type ." +

                        "?y p:subject ?id ." +

                        "?y p:offset ?offset ." +

                        "?y p:length ?length ." +

                        "?y p:exact ?exact ." +

                        "?y p:detection ?context ." +

                        " OPTIONAL { ?z p:subject ?id . ?z p:relevance ?score . } " +

                        // get disambiguated entity references if available

                        " OPTIONAL { ?did p:subject ?id . ?did p:name ?name . ?did rdf:type ?dtype . } " +

                        "FILTER (" +

                        "?type = t:Person || " +

                        "?type = t:City || " +

                        "?type = t:Continent || " +

                        "?type = t:Country || " +

                        "?type = t:ProvinceOrState || " +

                        "?type = t:Region || " +

                        "?type = t:Company || " +

                        "?type = t:Facility || " +

                        "?type = t:Organization " +

                        ")" +

                        "} ";

        Collection<CalaisEntityOccurrence> result = new ArrayList<CalaisEntityOccurrence>();

        try {

            SelectQuery sQuery = (SelectQuery) QueryParser.getInstance().parse(query);

            ResultSet rs = tcManager.executeSparqlQuery(sQuery, model);

            while (rs.hasNext()) {

                SolutionMapping row = rs.next();

                CalaisEntityOccurrence occ = new CalaisEntityOccurrence();

                Resource disambiguated = row.get("did");

                occ.id = (disambiguated == null ? row.get("id") : disambiguated);

                if (onlyNERMode) {

                    occ.type = row.get("type");

                }

                else {

                    occ.type = (disambiguated == null ? row.get("type") : row.get("dtype"));

                }

                if (calaisTypeMap != null) {

                    UriRef mappedType = calaisTypeMap.get(occ.type);

                    if (mappedType != null) {

                        occ.type = mappedType;

                    }

                }

                occ.name = ((Literal)row.get("name")).getLexicalForm();

                occ.exact = ((Literal)row.get("exact")).getLexicalForm();

                //TODO for html the offsets might not be those of the original document but refer to a cleaned up version?

                occ.offset = Integer.valueOf(((Literal) row.get("offset")).getLexicalForm());

                // remove brackets

                occ.context = ((Literal) row.get("context")).getLexicalForm().replaceAll("[\\[\\]]", "");

                occ.length = Integer.valueOf(((Literal) row.get("length")).getLexicalForm());

                if (row.get("score") != null) {

                    occ.relevance = Double.valueOf(((Literal) row.get("score")).getLexicalForm());

                }

                result.add(occ);

            }

        } catch (ParseException e) {

            throw new EngineException("Unable to parse SPARQL query for processing OpenCalais results",e);

        }

        log.info("Found {} occurences", result.size());

        return result;

    }

    /**

     * Sends a POST request to the given url.

     *

     * @param targetUrl a <code>String</code> with the target url

     * @param params a <code>Map<String,String></code> object containing the url parameters;

     *        use <code>null</code> if there are no parameters

     * @param body a <code>String</code> with the body of the post request; use

     *        <code>null</code> if the body is empty

     * @param contentType a <code>String</code> with the content type of the post

     *        request; use <code>null</code> for the default content type

     *        <code>text/xml; charset=utf-8</code>

     * @param responseEncoding a <code>String</code> with the encoding used to

     *        read the server response; use <code>null</code> for the default charset

     *

     * @return a <code>String</code> with the server response

     *

     * @throws IOException if an error occurs

     */

    public static String doPostRequest(

            String targetUrl, Map<String, String> params, String body, String contentType,

            String responseEncoding)

            throws IOException {

        StringBuilder urlString = new StringBuilder(targetUrl);

        // add parameters to url

        if (params != null) {

            if (!params.isEmpty()) {

                urlString.append("?");

            }

            Iterator<Map.Entry<String, String>> it = params.entrySet().iterator();

            while (it.hasNext()) {

                Map.Entry<String, String> oneParam = it.next();

                urlString

                        .append(oneParam.getKey()).append("=")

                        .append(URLEncoder.encode(oneParam.getValue(), "UTF-8"));

                if (it.hasNext()) {

                    urlString.append("&");

                }

            }

        }

        // init connection

        URL url = new URL(urlString.toString());

        HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();

        urlConn.setRequestMethod("POST");

        urlConn.setDoInput(true);

        if (null != body) {

            urlConn.setDoOutput(true);

        } else {

            urlConn.setDoOutput(false);

        }

        urlConn.setUseCaches(false);

        if (null == contentType) {

            contentType = "text/xml; charset=utf-8";

        }

        urlConn.setRequestProperty("Content-Type", contentType);

        // send POST output

        if (null != body) {

            OutputStreamWriter printout = new OutputStreamWriter(

                    urlConn.getOutputStream(), "UTF8");

            printout.write(body);

            printout.flush();

            printout.close();

        }

        // get response data

        if (null == responseEncoding) {

            responseEncoding = Charset.defaultCharset().toString();

        }

        return IOUtils.toString(

                urlConn.getInputStream(), responseEncoding);

    }

    /**

     * The activate method.

     *

     * @param ce the {@link ComponentContext}

     */

    protected void activate(ComponentContext ce) throws ConfigurationException {

        super.activate(ce);

        this.bundleContext = ce.getBundleContext();

        //TODO initialize Extractor

        Dictionary<String, Object> properties = ce.getProperties();

        String license = (String)properties.get(LICENSE_KEY);

        String url = (String)properties.get(CALAIS_URL_KEY);

        calaisTypeMapFile = (String)properties.get(CALAIS_TYPE_MAP_KEY);

        String standAlone = (String)properties.get(CALAIS_NER_ONLY_MODE_KEY);

        setLicenseKey(license);

        setCalaisUrl(url);

        calaisTypeMap = new HashMap<UriRef,UriRef>();

        loadTypeMap(calaisTypeMapFile);

        onlyNERMode = Boolean.parseBoolean(standAlone);

        //      this.tcManager = TcManager.getInstance();

    }

    /**

     * The deactivate method.

     *

     * @param ce the {@link ComponentContext}

     */

    protected void deactivate(ComponentContext ce) {

        super.deactivate(ce);

    }

}