Examples of BulkRequestBuilder


Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

    if (scrollResp.getHits().getTotalHits() > 0) {
      if (isClosed())
        throw new InterruptedException("Interrupted because River is closed");
      scrollResp = esIntegrationComponent.executeESScrollSearchNextRequest(scrollResp);
      BulkRequestBuilder esBulk = esIntegrationComponent.prepareESBulkRequestBuilder();
      while (scrollResp.getHits().getHits().length > 0) {
        for (SearchHit hit : scrollResp.getHits()) {
          logger.debug("Go to delete indexed issue for document id {}", hit.getId());
          if (jiraIssueIndexStructureBuilder.deleteIssueDocument(esBulk, hit)) {
            indexingInfo.issuesDeleted++;
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

        List<ElasticsearchObject> records = buffer.getRecords();
        if (records.isEmpty()) {
            return Collections.emptyList();
        }

        BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
        for (ElasticsearchObject record : records) {
            IndexRequestBuilder indexRequestBuilder =
                    elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
            indexRequestBuilder.setSource(record.getSource());
            Long version = record.getVersion();
            if (version != null) {
                indexRequestBuilder.setVersion(version);
            }
            Long ttl = record.getTtl();
            if (ttl != null) {
                indexRequestBuilder.setTTL(ttl);
            }
            Boolean create = record.getCreate();
            if (create != null) {
                indexRequestBuilder.setCreate(create);
            }
            bulkRequest.add(indexRequestBuilder);
        }

        while (true) {
            try {
                BulkResponse bulkResponse = bulkRequest.execute().actionGet();

                BulkItemResponse[] responses = bulkResponse.getItems();
                List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
                int numberOfSkippedRecords = 0;
                for (int i = 0; i < responses.length; i++) {
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

                module,
                System.currentTimeMillis()
            );

            // Bulk index request builder
            BulkRequestBuilder bulkBuilder = support.client().prepareBulk();

            // Handle changed entity states
            for( EntityState changedState : changedStates )
            {
                if( changedState.entityDescriptor().queryable() )
                {
                    switch( changedState.status() )
                    {
                        case REMOVED:
                            LOGGER.trace( "Removing Entity State from Index: {}", changedState );
                            remove( bulkBuilder, changedState.identity().identity() );
                            break;
                        case UPDATED:
                            LOGGER.trace( "Updating Entity State in Index: {}", changedState );
                            remove( bulkBuilder, changedState.identity().identity() );
                            String updatedJson = toJSON( changedState, newStates, uow );
                            LOGGER.trace( "Will index: {}", updatedJson );
                            index( bulkBuilder, changedState.identity().identity(), updatedJson );
                            break;
                        case NEW:
                            LOGGER.trace( "Creating Entity State in Index: {}", changedState );
                            String newJson = toJSON( changedState, newStates, uow );
                            LOGGER.trace( "Will index: {}", newJson );
                            index( bulkBuilder, changedState.identity().identity(), newJson );
                            break;
                        case LOADED:
                        default:
                            // Ignored
                            break;
                    }
                }
            }

            uow.discard();

            if( bulkBuilder.numberOfActions() > 0 )
            {

                // Execute bulk actions
                BulkResponse bulkResponse = bulkBuilder.execute().actionGet();

                // Handle errors
                if( bulkResponse.hasFailures() )
                {
                    throw new ElasticSearchIndexException( bulkResponse.buildFailureMessage() );
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

        riverConfig.getClient().prepareIndex(index, type, id).setSource(source)
                .setOpType(opType).setRefresh(true).execute().actionGet();
    }

    protected <T> void insertAll(final List<T> list, final OpType opType) {
        final BulkRequestBuilder bulkRequest = riverConfig.getClient()
                .prepareBulk();
        for (final T target : list) {
            final String id = getId(getSessionId(target), getUrl(target));
            final XContentBuilder source = getXContentBuilder(target);
            bulkRequest.add(riverConfig.getClient()
                    .prepareIndex(index, type, id).setSource(source)
                    .setOpType(opType));
        }
        final BulkResponse bulkResponse = bulkRequest.setRefresh(true)
                .execute().actionGet();
        if (bulkResponse.hasFailures()) {
            throw new RobotSystemException(bulkResponse.buildFailureMessage());
        }
    }
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

        // If there is nothing in the queues, just stop here
        if (toIndex.isEmpty() && toDelete.isEmpty()) {
            return null;
        }

        BulkRequestBuilder bulkRequestBuilder = elasticSearchClient.prepareBulk();
        //bulkRequestBuilder.setRefresh(true);

        // Execute index requests
        for (Map.Entry<IndexEntityKey, Object> entry : toIndex.entrySet()) {
            SearchableClassMapping scm = elasticSearchContextHolder.getMappingContextByType(entry.getKey().getClazz());
            if (isNewSession) {
                persistenceInterceptor = createInterceptor();
                session = SessionFactoryUtils.getSession(sessionFactory, true);
            }
            try {
                Object entity = entry.getValue();

                // If this not a transient instance, reattach it to the session
                if (session.contains(entity)) {
                    session.lock(entity, LockMode.NONE);
                    LOG.debug("Reattached entity to session");
                }

                XContentBuilder json = toJSON(entity);

                bulkRequestBuilder.add(
                        elasticSearchClient.prepareIndex()
                                .setIndex(scm.getIndexName())
                                .setType(scm.getElasticTypeName())
                                .setId(entry.getKey().getId()) // TODO : Composite key ?
                                .setSource(json)
                );
                if (LOG.isDebugEnabled()) {
                    try {
                        LOG.debug("Indexing " + entry.getKey().getClazz() + "(index:" + scm.getIndexName() + ",type:" + scm.getElasticTypeName() +
                                ") of id " + entry.getKey().getId() + " and source " + json.string());
                    } catch (IOException e) {
                    }
                }
            } finally {
                if (null != persistenceInterceptor) {
                    persistenceInterceptor.destroy();
                }
            }
        }

        // Execute delete requests
        for (IndexEntityKey key : toDelete) {
            SearchableClassMapping scm = elasticSearchContextHolder.getMappingContextByType(key.getClazz());
            if (LOG.isDebugEnabled()) {
                LOG.debug("Deleting object from index " + scm.getIndexName() + " and type " + scm.getElasticTypeName() + " and ID " + key.getId());
            }
            bulkRequestBuilder.add(
                    elasticSearchClient.prepareDelete()
                            .setIndex(scm.getIndexName())
                            .setType(scm.getElasticTypeName())
                            .setId(key.getId())
            );
        }

        // Perform bulk request
        OperationBatch completeListener = null;
        if (bulkRequestBuilder.numberOfActions() > 0) {
            synchronized(this) {
                completeListener = new OperationBatch(0, toIndex, toDelete);
                operationBatchList.add(completeListener);
                try {
                    bulkRequestBuilder.execute().addListener(completeListener);
                } catch (Exception e) {
                    throw new IndexException("Failed to index/delete " + bulkRequestBuilder.numberOfActions(), e);
                }
            }
        }

        return completeListener;
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

        return collectedResults;
    }

    Collection<Integer> bulkUpdate(MySearchHits objects, String indexName,
            String newType, boolean withVersion) {
        BulkRequestBuilder brb = client.prepareBulk();
        for (MySearchHit hit : objects.getHits()) {
            if (hit.id() == null || hit.id().isEmpty()) {
                logger.warn("Skipped object without id when bulkUpdate:" + hit);
                continue;
            }

            try {
                IndexRequest indexReq = Requests.indexRequest(indexName).type(newType).id(hit.id()).source(hit.source());
                if (withVersion)
                    indexReq.version(hit.version());
                if (hit.parent() != null && !hit.parent().isEmpty()) {
                    indexReq.parent(hit.parent());
                }
                brb.add(indexReq);
            } catch (Exception ex) {
                logger.warn("Cannot add object:" + hit + " to bulkIndexing action." + ex.getMessage());
            }
        }
        if (brb.numberOfActions() > 0) {
            BulkResponse rsp = brb.execute().actionGet();
            if (rsp.hasFailures()) {
                List<Integer> list = new ArrayList<Integer>(rsp.getItems().length);
                for (BulkItemResponse br : rsp.getItems()) {
                    if (br.isFailed())
                        list.add(br.getItemId());
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

        int attempt = 0;

        BulkResponse response = null;
        do {
            // build the bulk request for this iteration
            BulkRequestBuilder bulkBuilder = client.prepareBulk();
            for (Entry<String,IndexRequest> entry : bulkIndexRequests.entrySet()) {
                bulkBuilder.add(entry.getValue());
            }
            for (Entry<String,DeleteRequest> entry : bulkDeleteRequests.entrySet()) {
                bulkBuilder.add(entry.getValue());
            }

            attempt++;
            result = new ArrayList<Object>();
            if(response != null) {
                // at least second time through
                try {
                    Thread.sleep(this.bulkIndexRetryWaitMs);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
            response = bulkBuilder.execute().actionGet();
            if(response != null) {
                for (BulkItemResponse bulkItemResponse : response.getItems()) {
                    if(!bulkItemResponse.isFailed()) {
                        String itemId = bulkItemResponse.getId();
                        String itemRev = revisions.get(itemId);
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

   */
  @Before
  public void setUp() throws Exception {
    ObjectMapper mapper = new ObjectMapper();

    BulkRequestBuilder brb = node.client().prepareBulk();

    for (int i = 0; i < 1000; i++) {
      Beer beer = BeerHelper.generate();
      IndexRequest irq = new IndexRequest("meal", "beer", "beer_" + i);
      String jsonString = mapper.writeValueAsString(beer);
      irq.source(jsonString);
      brb.add(irq);
    }

    BulkResponse br = brb.execute().actionGet();
    Assert.assertFalse(br.hasFailures());

    node.client().admin().indices().prepareRefresh().execute().actionGet();
  }
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

  /**
   * When we stop a test, we remove all data
   */
  @After
  public void tearDown() {
    BulkRequestBuilder brb = node.client().prepareBulk();

    for (int i = 0; i < 1000; i++) {
      DeleteRequest dr = new DeleteRequest("meal", "beer", "beer_" + i);
      brb.add(dr);
    }

    BulkResponse br = brb.execute().actionGet();
    Assert.assertFalse(br.hasFailures());
  }
View Full Code Here

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder

  public void indexOneThousandBeers() throws JsonGenerationException, JsonMappingException,
      IOException {

    ObjectMapper mapper = new ObjectMapper(); // create once, reuse

    BulkRequestBuilder brb = null;
   
    // TODO Create the bulk
    brb = node.client().prepareBulk();
    for (int i = 0; i < 1000; i++) {
      Beer beer = BeerHelper.generate();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.