files = Arrays.toString(indexShard.store().directory().listAll());
} catch (Throwable e1) {
files += " (failure=" + ExceptionsHelper.detailedMessage(e1) + ")";
}
if (indexShouldExists && indexShard.indexService().store().persistent()) {
throw new IndexShardGatewayRecoveryException(shardId(), "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files, e);
}
}
if (si != null) {
if (indexShouldExists) {
version = si.getVersion();
/**
* We generate the translog ID before each lucene commit to ensure that
* we can read the current translog ID safely when we recover. The commits metadata
* therefor contains always the current / active translog ID.
*/
if (si.getUserData().containsKey(Translog.TRANSLOG_ID_KEY)) {
translogId = Long.parseLong(si.getUserData().get(Translog.TRANSLOG_ID_KEY));
} else {
translogId = version;
}
logger.trace("using existing shard data, translog id [{}]", translogId);
} else {
// it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
// its a "new index create" API, we have to do something, so better to clean it than use same data
logger.trace("cleaning existing shard, shouldn't exists");
IndexWriter writer = new IndexWriter(indexShard.store().directory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
writer.close();
}
}
} catch (Throwable e) {
throw new IndexShardGatewayRecoveryException(shardId(), "failed to fetch index version after copying it over", e);
}
recoveryState.getIndex().updateVersion(version);
recoveryState.getIndex().time(System.currentTimeMillis() - recoveryState.getIndex().startTime());
// since we recover from local, just fill the files and size
try {
int numberOfFiles = 0;
long totalSizeInBytes = 0;
for (String name : indexShard.store().directory().listAll()) {
numberOfFiles++;
long length = indexShard.store().directory().fileLength(name);
totalSizeInBytes += length;
recoveryState.getIndex().addFileDetail(name, length, length);
}
RecoveryState.Index index = recoveryState.getIndex();
index.totalFileCount(numberOfFiles);
index.totalByteCount(totalSizeInBytes);
index.reusedFileCount(numberOfFiles);
index.reusedByteCount(totalSizeInBytes);
index.recoveredFileCount(numberOfFiles);
index.recoveredByteCount(totalSizeInBytes);
} catch (Exception e) {
// ignore
}
recoveryState.getStart().startTime(System.currentTimeMillis());
recoveryState.setStage(RecoveryState.Stage.START);
if (translogId == -1) {
// no translog files, bail
indexShard.postRecovery("post recovery from gateway, no translog");
// no index, just start the shard and bail
recoveryState.getStart().time(System.currentTimeMillis() - recoveryState.getStart().startTime());
recoveryState.getStart().checkIndexTime(indexShard.checkIndexTook());
return;
}
// move an existing translog, if exists, to "recovering" state, and start reading from it
FsTranslog translog = (FsTranslog) indexShard.translog();
String translogName = "translog-" + translogId;
String recoverTranslogName = translogName + ".recovering";
File recoveringTranslogFile = null;
for (File translogLocation : translog.locations()) {
File tmpRecoveringFile = new File(translogLocation, recoverTranslogName);
if (!tmpRecoveringFile.exists()) {
File tmpTranslogFile = new File(translogLocation, translogName);
if (tmpTranslogFile.exists()) {
for (int i = 0; i < RECOVERY_TRANSLOG_RENAME_RETRIES; i++) {
if (tmpTranslogFile.renameTo(tmpRecoveringFile)) {
recoveringTranslogFile = tmpRecoveringFile;
break;
}
}
}
} else {
recoveringTranslogFile = tmpRecoveringFile;
break;
}
}
if (recoveringTranslogFile == null || !recoveringTranslogFile.exists()) {
// no translog to recovery from, start and bail
// no translog files, bail
indexShard.postRecovery("post recovery from gateway, no translog");
// no index, just start the shard and bail
recoveryState.getStart().time(System.currentTimeMillis() - recoveryState.getStart().startTime());
recoveryState.getStart().checkIndexTime(indexShard.checkIndexTook());
return;
}
// recover from the translog file
indexShard.performRecoveryPrepareForTranslog();
recoveryState.getStart().time(System.currentTimeMillis() - recoveryState.getStart().startTime());
recoveryState.getStart().checkIndexTime(indexShard.checkIndexTook());
recoveryState.getTranslog().startTime(System.currentTimeMillis());
recoveryState.setStage(RecoveryState.Stage.TRANSLOG);
StreamInput in = null;
final Set<String> typesToUpdate = Sets.newHashSet();
try {
TranslogStream stream = TranslogStreams.translogStreamFor(recoveringTranslogFile);
try {
in = stream.openInput(recoveringTranslogFile);
} catch (TruncatedTranslogException e) {
// file is empty or header has been half-written and should be ignored
logger.trace("ignoring truncation exception, the translog is either empty or half-written ([{}])", e.getMessage());
}
while (true) {
if (in == null) {
break;
}
Translog.Operation operation;
try {
if (stream instanceof LegacyTranslogStream) {
in.readInt(); // ignored opSize
}
operation = stream.read(in);
} catch (EOFException e) {
// ignore, not properly written the last op
logger.trace("ignoring translog EOF exception, the last operation was not properly written ([{}])", e.getMessage());
break;
} catch (IOException e) {
// ignore, not properly written last op
logger.trace("ignoring translog IO exception, the last operation was not properly written ([{}])", e.getMessage());
break;
}
try {
Engine.IndexingOperation potentialIndexOperation = indexShard.performRecoveryOperation(operation);
if (potentialIndexOperation != null && potentialIndexOperation.parsedDoc().mappingsModified()) {
if (!typesToUpdate.contains(potentialIndexOperation.docMapper().type())) {
typesToUpdate.add(potentialIndexOperation.docMapper().type());
}
}
recoveryState.getTranslog().addTranslogOperations(1);
} catch (ElasticsearchException e) {
if (e.status() == RestStatus.BAD_REQUEST) {
// mainly for MapperParsingException and Failure to detect xcontent
logger.info("ignoring recovery of a corrupt translog entry", e);
} else {
throw e;
}
}
}
} catch (Throwable e) {
// we failed to recovery, make sure to delete the translog file (and keep the recovering one)
indexShard.translog().closeWithDelete();
throw new IndexShardGatewayRecoveryException(shardId, "failed to recover shard", e);
} finally {
IOUtils.closeWhileHandlingException(in);
}
indexShard.performRecoveryFinalization(true);