/*
* Copyright (c) 2011 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.flaptor.indextank;
import org.apache.log4j.Logger;
import com.flaptor.indextank.index.Document;
import com.flaptor.indextank.index.IndexEngine;
import com.flaptor.indextank.rpc.LogRecord;
import com.flaptor.indextank.storage.LogStorageIndexReader;
import com.flaptor.util.Execute;
public class LogIndexRecoverer implements Runnable {
private static final Logger logger = Logger.getLogger(Execute.whoAmI());
private IndexEngine index;
private final String indexCode;
private final String logServerHost;
private final int logServerPort;
public LogIndexRecoverer(IndexEngine index, String indexCode, String logServerHost, int logServerPort) {
this.index = index;
this.indexCode = indexCode;
this.logServerHost = logServerHost;
this.logServerPort = logServerPort;
}
@Override
public void run() {
BoostingIndexer indexer = index.getIndexer();
logger.info("Recovering index from log based storage.");
int count = 0;
LogStorageIndexReader reader = new LogStorageIndexReader(logServerHost, logServerPort, 5, indexCode);
try {
logger.info("Starting recovery from Log Based Storage");
for (LogRecord record : reader) {
if (record.is_deleted()) {
indexer.del(record.get_docid());
}
if (record.get_fields_size() > 0) {
if (!record.get_fields().containsKey("timestamp")) {
logger.warn("Document " + record.get_docid() + " had no timestamp, skipping it.");
continue;
}
Integer timestamp;
try {
timestamp = Integer.valueOf(record.get_fields().get("timestamp"));
} catch (NumberFormatException e) {
logger.warn("Document " + record.get_docid() + " had an invalid timestamp '" + record.get_fields().get("timestamp") + "', skipping it.");
continue;
}
Document document = new Document(record.get_fields());
indexer.add(record.get_docid(), document, timestamp, record.get_variables());
} else {
indexer.updateBoosts(record.get_docid(), record.get_variables());
}
if (record.is_set_categories()) {
indexer.updateCategories(record.get_docid(), record.get_categories());
}
count++;
if (count % 1000 == 0) {
logger.info("Already recovered " + count + " documents");
}
}
} catch (Exception ex) {
throw new RuntimeException("Something BAD happened: ",ex);
}
}
}