/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.river.mongodb.gridfs;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.client.Requests.countRequest;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import java.util.List;
import org.bson.types.ObjectId;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.WriteConcern;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSInputFile;
public class RiverMongoGridFSBulkImportTest extends RiverMongoGridFSTestAbstract {
private static final String TEST_ATTACHMENT_HTML = "/org/elasticsearch/river/mongodb/gridfs/test-attachment.html";
private static final String CRITERIA_ATTACHMENT_HTML = "Aliquam";
private static final String TEST_ATTACHMENT_PDF = "/org/elasticsearch/river/mongodb/gridfs/lorem.pdf";
private static final String CRITERIA_ATTACHMENT_PDF = "Lorem ipsum dolor";
private static final String TEST_ATTACHMENT_PDF_2 = "/org/elasticsearch/river/mongodb/gridfs/test-document.pdf";
private static final String CRITERIA_ATTACHMENT_PDF_2 = "Design and Specifications";
private enum SampleFile {
HTML(TEST_ATTACHMENT_HTML, CRITERIA_ATTACHMENT_HTML), PDF(TEST_ATTACHMENT_PDF, CRITERIA_ATTACHMENT_PDF), PDF2(
TEST_ATTACHMENT_PDF_2, CRITERIA_ATTACHMENT_PDF_2), ;
private final String filename;
private final String criteria;
private SampleFile(String filename, String criteria) {
this.filename = filename;
this.criteria = criteria;
}
public String getFilename() {
return filename;
}
public String getCriteria() {
return criteria;
}
}
private DB mongoDB;
private DBCollection mongoCollection;
@BeforeClass
public void createDatabase() {
logger.debug("createDatabase {}", getDatabase());
try {
mongoDB = getMongo().getDB(getDatabase());
mongoDB.setWriteConcern(WriteConcern.REPLICAS_SAFE);
super.createRiver(TEST_MONGODB_RIVER_GRIDFS_JSON);
logger.info("Start createCollection");
mongoCollection = mongoDB.createCollection(getCollection(), null);
Assert.assertNotNull(mongoCollection);
} catch (Throwable t) {
logger.error("createDatabase failed.", t);
}
}
@AfterClass
public void cleanUp() {
super.deleteRiver();
logger.info("Drop database " + mongoDB.getName());
mongoDB.dropDatabase();
}
@Test(enabled = false)
public void testImportPDFAttachment() throws Exception {
logger.debug("*** testImportPDFAttachment ***");
long count = 10000;
List<String> ids = newArrayList();
SampleFile attachment = SampleFile.PDF2;
final byte[] content = copyToBytesFromClasspath(attachment.getFilename());
logger.debug("Content in bytes: {}", content.length);
GridFS gridFS = new GridFS(mongoDB);
for (int i = 0; i < count; i++) {
GridFSInputFile in = gridFS.createFile(content);
in.setFilename("lorem-" + i + ".pdf");
in.setContentType("application/pdf");
in.save();
in.validate();
String id = in.getId().toString();
logger.debug("GridFS in: {}", in);
logger.debug("Document created with id: {}", id);
GridFSDBFile out = gridFS.findOne(in.getFilename());
logger.debug("GridFS from findOne: {}", out);
out = gridFS.findOne(new ObjectId(id));
logger.debug("GridFS from findOne: {}", out);
Assert.assertEquals(out.getId(), in.getId());
ids.add(id);
}
CountResponse countResponse;
while (true) {
Thread.sleep(wait);
refreshIndex();
countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.debug("Index total count: {}", countResponse.getCount());
if (countResponse.getCount() == count) {
break;
}
}
countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.debug("Index total count: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(count));
SearchResponse response = getNode().client().prepareSearch(getIndex())
.setQuery(QueryBuilders.queryString(attachment.getCriteria())).execute().actionGet();
long totalHits = response.getHits().getTotalHits();
logger.debug("TotalHits: {}", totalHits);
assertThat(totalHits, equalTo(count));
for (String id : ids) {
gridFS.remove(new ObjectId(id));
}
Thread.sleep(wait);
refreshIndex();
countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.debug("Count after delete request: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(0L));
}
}