@Override
public void write(NutchDocument doc) throws IOException {
String id = (String)doc.getFieldValue("url");
String type = doc.getDocumentMeta().get("type");
if (type == null) type = "doc";
IndexRequestBuilder request = client.prepareIndex(defaultIndex, type, id);
Map<String, Object> source = new HashMap<String, Object>();
// Loop through all fields of this doc
for (String fieldName : doc.getFieldNames()) {
if (doc.getField(fieldName).getValues().size() > 1) {
source.put(fieldName, doc.getFieldValue(fieldName));
// Loop through the values to keep track of the size of this document
for (Object value : doc.getField(fieldName).getValues()) {
bulkLength += value.toString().length();
}
} else {
source.put(fieldName, doc.getFieldValue(fieldName));
bulkLength += doc.getFieldValue(fieldName).toString().length();
}
}
request.setSource(source);
// Add this indexing request to a bulk request
bulk.add(request);
indexedDocs++;
bulkDocs++;