HttpResponseContent response = performHttpGetCall(url, null);
if (response.contentType != null && response.contentType.contains("text/html")) {
try {
Document doc = Jsoup.parse(new ByteArrayInputStream(response.content), null, url);
if (htmlMapping == null) {
return doc.html();
} else {
Map<String, String> ret = new HashMap<>();
for (String dataField : htmlMapping.keySet()) {
String value = null;
Map<String, Object> fieldMappingConfig = htmlMapping.get(dataField);
String cssSelector = Utils.trimToNull((String) fieldMappingConfig.get(CFG_HM_CSS_SELECTOR));
boolean stripHtml = XContentMapValues.nodeBooleanValue(fieldMappingConfig.get(CFG_HM_STRIP_HTML), false);
if (cssSelector != null) {
Elements elements = doc.select(cssSelector);
if (elements != null && !elements.isEmpty()) {
String valueAttribute = Utils.trimToNull((String) fieldMappingConfig.get(CFG_HM_VALUE_ATTRIBUTE));
if (valueAttribute != null) {
StringBuilder sb = new StringBuilder();
for (Element e : elements) {
String v = Utils.trimToNull(e.attr(valueAttribute));
if (v != null) {
if (sb.length() > 0) {
sb.append(" ");
}
sb.append(v);
}
}
value = Utils.trimToNull(sb.toString());
} else {
if (stripHtml) {
value = convertElementsToText(elements);
} else {
if (elements.size() == 1) {
value = elements.html();
} else {
value = elements.outerHtml();
}
}
}
}
} else {
if (stripHtml) {
value = convertNodeToText(doc);
} else {
value = doc.html();
}
}
ret.put(dataField, value);
}
return ret;