/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.waveprotocol.box.server.waveserver;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListenableFutureTask;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.http.HttpStatus;
import org.waveprotocol.box.common.DeltaSequence;
import org.waveprotocol.box.common.Snippets;
import org.waveprotocol.box.server.CoreSettings;
import org.waveprotocol.box.server.executor.ExecutorAnnotations.SolrExecutor;
import org.waveprotocol.box.server.robots.util.ConversationUtil;
import org.waveprotocol.box.stat.Timed;
import org.waveprotocol.wave.model.document.operation.DocInitialization;
import org.waveprotocol.wave.model.id.IdUtil;
import org.waveprotocol.wave.model.id.WaveletName;
import org.waveprotocol.wave.model.version.HashedVersion;
import org.waveprotocol.wave.model.wave.ParticipantId;
import org.waveprotocol.wave.model.wave.data.ReadableBlipData;
import org.waveprotocol.wave.model.wave.data.ReadableWaveletData;
import org.waveprotocol.wave.model.waveref.WaveRef;
import org.waveprotocol.wave.util.escapers.jvm.JavaWaverefEncoder;
import org.waveprotocol.wave.util.logging.Log;
import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.logging.Level;
/**
* @author Frank R. <renfeng.cn@gmail.com>
*/
@Singleton
public class SolrWaveIndexerImpl extends AbstractWaveIndexer implements WaveBus.Subscriber,
PerUserWaveViewBus.Listener {
private static final Log LOG = Log.get(SolrWaveIndexerImpl.class);
private final Executor executor;
private final ReadableWaveletDataProvider waveletDataProvider;
private final String solrBaseUrl;
@Inject
public SolrWaveIndexerImpl(WaveMap waveMap, WaveletProvider waveletProvider,
ReadableWaveletDataProvider waveletDataProvider, ConversationUtil conversationUtil,
WaveletNotificationDispatcher notificationDispatcher,
@Named(CoreSettings.SOLR_BASE_URL) String solrUrl,
@SolrExecutor Executor solrExecutor) {
super(waveMap, waveletProvider);
executor = solrExecutor;
solrBaseUrl = solrUrl;
this.waveletDataProvider = waveletDataProvider;
notificationDispatcher.subscribe(this);
}
@Override
public ListenableFuture<Void> onParticipantAdded(final WaveletName waveletName,
ParticipantId participant) {
/*
* ignored. See waveletCommitted(WaveletName, HashedVersion)
*/
return null;
}
@Override
public ListenableFuture<Void> onParticipantRemoved(final WaveletName waveletName,
ParticipantId participant) {
/*
* ignored. See waveletCommitted(WaveletName, HashedVersion)
*/
return null;
}
@Override
public ListenableFuture<Void> onWaveInit(final WaveletName waveletName) {
ListenableFutureTask<Void> task = ListenableFutureTask.create(new Callable<Void>() {
@Override
public Void call() throws Exception {
ReadableWaveletData waveletData;
try {
waveletData = waveletDataProvider.getReadableWaveletData(waveletName);
updateIndex(waveletData);
} catch (WaveServerException e) {
LOG.log(Level.SEVERE, "Failed to initialize index for " + waveletName, e);
throw e;
}
return null;
}
});
executor.execute(task);
return task;
}
@Override
protected void processWavelet(WaveletName waveletName) {
onWaveInit(waveletName);
}
@Override
protected void postIndexHook() {
try {
getWaveMap().unloadAllWavelets();
} catch (WaveletStateException e) {
throw new IndexException("Problem encountered while cleaning up", e);
}
}
@Timed
private void updateIndex(ReadableWaveletData wavelet) throws IndexException {
Preconditions.checkNotNull(wavelet);
if (IdUtil.isConversationalId(wavelet.getWaveletId())) {
JsonArray docsJson = buildJsonDoc(wavelet);
postUpdateToSolr(wavelet, docsJson);
}
}
private void postUpdateToSolr(ReadableWaveletData wavelet, JsonArray docsJson) {
PostMethod postMethod =
new PostMethod(solrBaseUrl + "/update/json?commit=true");
try {
RequestEntity requestEntity =
new StringRequestEntity(docsJson.toString(), "application/json", "UTF-8");
postMethod.setRequestEntity(requestEntity);
HttpClient httpClient = new HttpClient();
int statusCode = httpClient.executeMethod(postMethod);
if (statusCode != HttpStatus.SC_OK) {
throw new IndexException(wavelet.getWaveId().serialise());
}
} catch (IOException e) {
throw new IndexException(String.valueOf(wavelet.getWaveletId()), e);
} finally {
postMethod.releaseConnection();
}
}
JsonArray buildJsonDoc(ReadableWaveletData wavelet) {
JsonArray docsJson = new JsonArray();
String waveletId = wavelet.getWaveletId().serialise();
String modified = Long.toString(wavelet.getLastModifiedTime());
String creator = wavelet.getCreator().getAddress();
for (String docName : wavelet.getDocumentIds()) {
ReadableBlipData document = wavelet.getDocument(docName);
if (!IdUtil.isBlipId(docName)) {
continue;
}
Iterable<DocInitialization> ops = Lists.newArrayList(document.getContent().asOperation());
String text = Snippets.collateTextForOps(ops, new Function<StringBuilder, Void>() {
@Override
public Void apply(StringBuilder resultBuilder) {
resultBuilder.append("\n");
return null;
}
});
JsonArray participantsJson = new JsonArray();
for (ParticipantId participant : wavelet.getParticipants()) {
String participantAddress = participant.toString();
participantsJson.add(new JsonPrimitive(participantAddress));
}
String id =
JavaWaverefEncoder.encodeToUriPathSegment(WaveRef.of(wavelet.getWaveId(),
wavelet.getWaveletId(), docName));
JsonObject docJson = new JsonObject();
docJson.addProperty(SolrSearchProviderImpl.ID, id);
docJson.addProperty(SolrSearchProviderImpl.WAVE_ID, wavelet.getWaveId().serialise());
docJson.addProperty(SolrSearchProviderImpl.WAVELET_ID, waveletId);
docJson.addProperty(SolrSearchProviderImpl.DOC_NAME, docName);
docJson.addProperty(SolrSearchProviderImpl.LMT, modified);
docJson.add(SolrSearchProviderImpl.WITH, participantsJson);
docJson.add(SolrSearchProviderImpl.WITH_FUZZY, participantsJson);
docJson.addProperty(SolrSearchProviderImpl.CREATOR, creator);
docJson.addProperty(SolrSearchProviderImpl.TEXT, text);
docJson.addProperty(SolrSearchProviderImpl.IN, "inbox");
docsJson.add(docJson);
}
return docsJson;
}
@Override
public void waveletUpdate(final ReadableWaveletData wavelet, DeltaSequence deltas) {
/*
* Overridden out for optimization, see waveletCommitted(WaveletName,
* HashedVersion)
*/
}
@Override
public void waveletCommitted(final WaveletName waveletName, final HashedVersion version) {
Preconditions.checkNotNull(waveletName);
ListenableFutureTask<Void> task = ListenableFutureTask.create(new Callable<Void>() {
@Override
public Void call() throws Exception {
ReadableWaveletData waveletData;
try {
waveletData = waveletDataProvider.getReadableWaveletData(waveletName);
LOG.fine("commit " + version + " " + waveletData.getVersion());
if (waveletData.getVersion() == version.getVersion()) {
updateIndex(waveletData);
}
} catch (WaveServerException e) {
LOG.log(Level.SEVERE, "Failed to update index for " + waveletName, e);
throw e;
}
return null;
}
});
executor.execute(task);
}
@Override
public synchronized void remakeIndex() throws WaveletStateException, WaveServerException {
/*-
* To fully rebuild the index, need to delete everything first
* the <query> tag should contain the value of
* org.waveprotocol.box.server.waveserver.SolrSearchProviderImpl.Q
*
* http://localhost:8983/solr/update?stream.body=<delete><query>waveId_s:[*%20TO%20*]%20AND%20waveletId_s:[*%20TO%20*]%20AND%20docName_s:[*%20TO%20*]%20AND%20lmt_l:[*%20TO%20*]%20AND%20with_ss:[*%20TO%20*]%20AND%20with_txt:[*%20TO%20*]%20AND%20creator_t:[*%20TO%20*]</query></delete>
* http://localhost:8983/solr/update?stream.body=<commit/>
*
* see
* http://wiki.apache.org/solr/FAQ#How_can_I_delete_all_documents_from_my_index.3F
*/
sendRequestToDeleteSolrIndex();
super.remakeIndex();
}
private void sendRequestToDeleteSolrIndex() {
GetMethod getMethod = new GetMethod();
try {
getMethod
.setURI(new URI(solrBaseUrl + "/update?wt=json"
+ "&stream.body=<delete><query>" + SolrSearchProviderImpl.Q + "</query></delete>",
false));
HttpClient httpClient = new HttpClient();
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode == HttpStatus.SC_OK) {
getMethod.setURI(new URI(solrBaseUrl + "/update?wt=json"
+ "&stream.body=<commit/>", false));
httpClient = new HttpClient();
statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
LOG.warning("failed to clean solr index");
}
} else {
LOG.warning("failed to clean solr index");
}
} catch (Exception e) {
LOG.warning("failed to clean solr index", e);
} finally {
getMethod.releaseConnection();
}
}
}