Examples of BulkRequestBuilder

com.dotcms.repackage.org.elasticsearch.action.bulk.BulkRequestBuilder
org.elasticsearch.action.bulk.BulkRequestBuilder
A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes it in a single batch.
org.elasticsearch.client.action.bulk.BulkRequestBuilder
A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes it in a single batch. @author kimchy (shay.banon)

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;


                    XContentBuilder builder = jsonBuilder().startObject();
                    builder.field("id", Integer.toString(counter));
                    builder.field("s_value", sValues[counter % sValues.length]);
                    builder.field("l_value", lValues[counter % lValues.length]);


                    builder.startArray("sm_value");
                    for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
                        builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
                    }
                    builder.endArray();


                    builder.startArray("lm_value");
                    for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
                        builder.value(lValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
                    }
                    builder.endArray();


                    builder.endObject();


                    request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter))
                            .source(builder));
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;


                    XContentBuilder builder = jsonBuilder().startObject();
                    builder.field("id", Integer.toString(counter));
                    builder.field("l_value", lValues[counter % lValues.length]);


                    builder.endObject();


                    request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter))
                            .source(builder));
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

                    .setPercolate("color:green").execute().actionGet();
            assertThat(index.matches().size(), equalTo(0));
        }


        // test bulk
        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
        for (int i = 0; i < 10; i++) {
            bulkRequestBuilder.add(client.prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "value1")
                    .setPercolate("*"));
        }
        BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
        assertThat(bulkResponse.hasFailures(), equalTo(false));
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            IndexResponse index = bulkItemResponse.response();
            assertThat(index.matches().size(), equalTo(1));
            assertThat(index.matches(), hasItem("kuku"));

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

    @Override public DeleteRequestBuilder prepareDelete(String index, String type, String id) {
        return prepareDelete().setIndex(index).setType(type).setId(id);
    }


    @Override public BulkRequestBuilder prepareBulk() {
        return new BulkRequestBuilder(this);
    }

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

                    }


                    if (task != null && task.getBody() != null) {
                        final List<Long> deliveryTags = Lists.newArrayList();


                        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();


                        try {
                            bulkRequestBuilder.add(task.getBody(), 0, task.getBody().length, false);
                        } catch (Exception e) {
                            logger.warn("failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag());
                            try {
                                channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
                            } catch (IOException e1) {
                                logger.warn("failed to ack [{}]", e1, task.getEnvelope().getDeliveryTag());
                            }
                            continue;
                        }


                        deliveryTags.add(task.getEnvelope().getDeliveryTag());


                        if (bulkRequestBuilder.numberOfActions() < bulkSize) {
                            // try and spin some more of those without timeout, so we have a bigger bulk (bounded by the bulk size)
                            try {
                                while ((task = consumer.nextDelivery(bulkTimeout.millis())) != null) {
                                    try {
                                        bulkRequestBuilder.add(task.getBody(), 0, task.getBody().length, false);
                                    } catch (Exception e) {
                                        logger.warn("failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag());
                                        try {
                                            channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
                                        } catch (IOException e1) {
                                            logger.warn("failed to ack on failure [{}]", e1, task.getEnvelope().getDeliveryTag());
                                        }
                                    }
                                    deliveryTags.add(task.getEnvelope().getDeliveryTag());
                                    if (bulkRequestBuilder.numberOfActions() >= bulkSize) {
                                        break;
                                    }
                                }
                            } catch (InterruptedException e) {
                                if (closed) {
                                    break;
                                }
                            }
                        }


                        if (logger.isTraceEnabled()) {
                            logger.trace("executing bulk with [{}] actions", bulkRequestBuilder.numberOfActions());
                        }


                        if (ordered) {
                            try {
                                BulkResponse response = bulkRequestBuilder.execute().actionGet();
                                if (response.hasFailures()) {
                                    // TODO write to exception queue?
                                    logger.warn("failed to execute" + response.buildFailureMessage());
                                }
                                for (Long deliveryTag : deliveryTags) {
                                    try {
                                        channel.basicAck(deliveryTag, false);
                                    } catch (IOException e1) {
                                        logger.warn("failed to ack [{}]", e1, deliveryTag);
                                    }
                                }
                            } catch (Exception e) {
                                logger.warn("failed to execute bulk", e);
                            }
                        } else {
                            bulkRequestBuilder.execute(new ActionListener<BulkResponse>() {
                                @Override public void onResponse(BulkResponse response) {
                                    if (response.hasFailures()) {
                                        // TODO write to exception queue?
                                        logger.warn("failed to execute" + response.buildFailureMessage());
                                    }

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

                    if (closed) {
                        return;
                    }
                    continue;
                }
                BulkRequestBuilder bulk = client.prepareBulk();
                String lastSeq = null;
                String lineSeq = processLine(s, bulk);
                if (lineSeq != null) {
                    lastSeq = lineSeq;
                }


                // spin a bit to see if we can get some more changes
                try {
                    while ((s = stream.poll(bulkTimeout.millis(), TimeUnit.MILLISECONDS)) != null) {
                        lineSeq = processLine(s, bulk);
                        if (lineSeq != null) {
                            lastSeq = lineSeq;
                        }


                        if (bulk.numberOfActions() >= bulkSize) {
                            break;
                        }
                    }
                } catch (InterruptedException e) {
                    if (closed) {
                        return;
                    }
                }


                if (lastSeq != null) {
                    try {
                        if (logger.isTraceEnabled()) {
                            logger.trace("processing [_seq  ]: [{}]/[{}]/[{}], last_seq [{}]", riverIndexName, riverName.name(), "_seq", lastSeq);
                        }
                        bulk.add(indexRequest(riverIndexName).type(riverName.name()).id("_seq")
                                .source(jsonBuilder().startObject().startObject("couchdb").field("last_seq", lastSeq).endObject().endObject()));
                    } catch (IOException e) {
                        logger.warn("failed to add last_seq entry to bulk indexing");
                    }
                }


                try {
                    BulkResponse response = bulk.execute().actionGet();
                    if (response.hasFailures()) {
                        // TODO write to exception queue?
                        logger.warn("failed to execute" + response.buildFailureMessage());
                    }
                } catch (Exception e) {

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

        }
    }


    public void bulkUpdate(Collection<MyTweet> tweets, String indexName) {
        // now using bulk API instead of feeding each doc separate with feedDoc
        BulkRequestBuilder brb = client.prepareBulk();
        for (MyTweet tweet : tweets) {
            String id = Long.toString(tweet.getId());
            XContentBuilder source = createDoc(tweet);
            brb.add(Requests.indexRequest(indexName).type(getIndexType()).id(id).source(source));
        }
        if (brb.numberOfActions() > 0) {
//            System.out.println("actions:" + brb.numberOfActions());
            brb.execute().actionGet();
        }
    }

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

      throw new RuntimeException("bulkAddDocuments not supported on multi-index manager");
    }
    if (!docsJson.isJsonArray()) {
      throw new RuntimeException("bulkAddDocuments - not a list");
    }
    BulkRequestBuilder brb = _elasticClient.prepareBulk();
    
    JsonArray docJsonArray = docsJson.getAsJsonArray();
    for (JsonElement docJson: docJsonArray) {
      IndexRequest ir = new IndexRequest(_sIndexName);
      ir.type(_sIndexType);
      if (null != sParentId) {
        ir.parent(sParentId);
      }
      if (!bAllowOverwrite) {
        ir.opType(OpType.CREATE);
      }//TESTED
      
      // Some _id unpleasantness
      if (null != idFieldName) {
        
        String id = docJson.getAsJsonObject().get(idFieldName).getAsString();
        ir.id(id);
        ir.source(docJson.toString());
      }//TESTED
      
      brb.add(ir);
    }
    brb.setConsistencyLevel(WriteConsistencyLevel.ONE);
    return brb.execute().actionGet();
  }//TESTED (including children and id hashmap)

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

  public BulkResponse bulkAddDocuments(Collection<DBObject> docsJson, String idFieldName, String sParentId, boolean bAllowOverwrite)
  {
    if (null != _multiIndex) {
      throw new RuntimeException("bulkAddDocuments not supported on multi-index manager");
    }
    BulkRequestBuilder brb = _elasticClient.prepareBulk();
    
    for (DBObject docJson: docsJson) {
      IndexRequest ir = new IndexRequest(_sIndexName);
      ir.type(_sIndexType);
      if (null != sParentId) {
        ir.parent(sParentId);
      }
      if (!bAllowOverwrite) {
        ir.opType(OpType.CREATE);
      }//TESTED
      
      // Some _id unpleasantness
      if (null != idFieldName) {
        
        String id = (String) docJson.get(idFieldName);
        ir.id(id);
        ir.source(docJson.toString());
      }//TESTED
      
      brb.add(ir);
    }
    brb.setConsistencyLevel(WriteConsistencyLevel.ONE);
    return brb.execute().actionGet();
  }//TESTED (including children and id hashmap)

View Full Code Here

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder

  public BulkResponse bulkDeleteDocuments(List<String> ids, String sParentId)
  {
    if (null != _multiIndex) {
      throw new RuntimeException("bulkDeleteDocuments not supported on multi-index manager");
    }
    BulkRequestBuilder brb = _elasticClient.prepareBulk();
    for (String id: ids) {
      
      DeleteRequest dr = new DeleteRequest(_sIndexName, _sIndexType, id);
      if (null != sParentId) {
        dr.parent(sParentId);
      }
      brb.add(dr);
    }
    brb.setConsistencyLevel(WriteConsistencyLevel.ONE);
    
    return brb.execute().actionGet();
  }//TESTED (inc children)

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.