Package com.nardoz.restopengov.utils

Source Code of com.nardoz.restopengov.utils.ElasticIndexer

package com.nardoz.restopengov.utils;

import com.nardoz.restopengov.Crawler;
import com.typesafe.config.ConfigFactory;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;

public class ElasticIndexer implements ICSVFetcherResult {

    protected String index;
    protected String type;
    protected String id;
    protected Integer maxPerBulk;
    protected Integer itemCounter = 0;
    protected Client client;
    protected BulkRequestBuilder bulkRequest;

    public ElasticIndexer(Client client, String type, String id) {
        this.client = client;
        this.index = ConfigFactory.load().getString("restopengov.index");
        this.type = type;
        this.id = id;
        this.maxPerBulk = ConfigFactory.load().getInt("restopengov.max-per-bulk");
    }

    public void onStart() {
        itemCounter = 0;
        bulkRequest = client.prepareBulk();
        Crawler.logger.info("***** Started: " + index + "/" + type + "/" + id + "-[..] *****");
    }

    public void add(String uuid, String json) {

        if(itemCounter.equals(maxPerBulk)) {
            onEnd();
            onStart();
        }

        String resourceId = id + "-" + uuid;

        bulkRequest.add(client.prepareIndex(index, type, resourceId).setSource(json));
        itemCounter++;

        Crawler.logger.debug("Adding item #" + itemCounter + ": " + index + "/" + type + "/" + resourceId);
    }

    public void onEnd() {
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();

        if(bulkResponse.hasFailures()) {
            Crawler.logger.error("Elasticsearch Failure: " + bulkResponse.buildFailureMessage());
        } else {
            Crawler.logger.info("***** " + itemCounter + " items saved in: " + index + "/" + type + "/" + id + "-[..] *****");
        }
    }

}
TOP

Related Classes of com.nardoz.restopengov.utils.ElasticIndexer

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.