/*
*
* * Copyright 2014 Orient Technologies LTD (info(at)orientechnologies.com)
* *
* * Licensed under the Apache License, Version 2.0 (the "License");
* * you may not use this file except in compliance with the License.
* * You may obtain a copy of the License at
* *
* * http://www.apache.org/licenses/LICENSE-2.0
* *
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS,
* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* * See the License for the specific language governing permissions and
* * limitations under the License.
* *
* * For more information: http://www.orientechnologies.com
*
*/
package com.orientechnologies.orient.core.storage.impl.local.paginated;
import com.orientechnologies.common.io.OFileUtils;
import com.orientechnologies.common.io.OIOUtils;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.parser.OSystemVariableResolver;
import com.orientechnologies.orient.core.command.OCommandOutputListener;
import com.orientechnologies.orient.core.compression.impl.OZIPCompressionUtil;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.engine.local.OEngineLocalPaginated;
import com.orientechnologies.orient.core.exception.OStorageException;
import com.orientechnologies.orient.core.index.hashindex.local.cache.OReadWriteDiskCache;
import com.orientechnologies.orient.core.metadata.OMetadataDefault;
import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
import com.orientechnologies.orient.core.storage.impl.local.OFreezableStorage;
import com.orientechnologies.orient.core.storage.impl.local.OStorageConfigurationSegment;
import com.orientechnologies.orient.core.storage.impl.local.OStorageVariableParser;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.ODiskWriteAheadLog;
import com.orientechnologies.orient.core.util.OBackupable;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* @author Andrey Lomakin
* @since 28.03.13
*/
public class OLocalPaginatedStorage extends OAbstractPaginatedStorage implements OFreezableStorage, OBackupable {
private static final int ONE_KB = 1024;
private final int DELETE_MAX_RETRIES;
private final int DELETE_WAIT_TIME;
private final OStorageVariableParser variableParser;
private final OPaginatedStorageDirtyFlag dirtyFlag;
private String storagePath;
private ScheduledExecutorService fuzzyCheckpointExecutor;
private ExecutorService checkpointExecutor;
public OLocalPaginatedStorage(final String name, final String filePath, final String mode) throws IOException {
super(name, filePath, mode);
File f = new File(url);
if (f.exists() || !exists(f.getParent())) {
// ALREADY EXISTS OR NOT LEGACY
storagePath = OSystemVariableResolver.resolveSystemVariables(OFileUtils.getPath(new File(url).getPath()));
} else {
// LEGACY DB
storagePath = OSystemVariableResolver.resolveSystemVariables(OFileUtils.getPath(new File(url).getParent()));
}
storagePath = OIOUtils.getPathFromDatabaseName(storagePath);
variableParser = new OStorageVariableParser(storagePath);
configuration = new OStorageConfigurationSegment(this);
DELETE_MAX_RETRIES = OGlobalConfiguration.FILE_DELETE_RETRY.getValueAsInteger();
DELETE_WAIT_TIME = OGlobalConfiguration.FILE_DELETE_DELAY.getValueAsInteger();
dirtyFlag = new OPaginatedStorageDirtyFlag(storagePath + File.separator + "dirty.fl");
}
@Override
public void create(final Map<String, Object> iProperties) {
final File storageFolder = new File(storagePath);
if (!storageFolder.exists())
storageFolder.mkdirs();
super.create(iProperties);
}
public boolean exists() {
return exists(storagePath);
}
@Override
public String getURL() {
return OEngineLocalPaginated.NAME + ":" + url;
}
public String getStoragePath() {
return storagePath;
}
public OStorageVariableParser getVariableParser() {
return variableParser;
}
public void scheduleFullCheckpoint() {
if (checkpointExecutor != null)
checkpointExecutor.execute(new Runnable() {
@Override
public void run() {
try {
makeFullCheckpoint();
} catch (Throwable t) {
OLogManager.instance().error(this, "Error during background checkpoint creation for storage " + name, t);
}
}
});
}
@Override
public String getType() {
return OEngineLocalPaginated.NAME;
}
@Override
public void backup(OutputStream out, Map<String, Object> options, final Callable<Object> callable,
final OCommandOutputListener iOutput, final int compressionLevel, final int bufferSize) throws IOException {
freeze(false);
try {
if (callable != null)
try {
callable.call();
} catch (Exception e) {
OLogManager.instance().error(this, "Error on callback invocation during backup", e);
}
final OutputStream bo = bufferSize > 0 ? new BufferedOutputStream(out, bufferSize) : out;
try {
OZIPCompressionUtil.compressDirectory(new File(getStoragePath()).getAbsolutePath(), bo, new String[] { ".wal" }, iOutput,
compressionLevel);
} finally {
if (bufferSize > 0) {
bo.flush();
bo.close();
}
}
} finally {
release();
}
}
@Override
public void restore(InputStream in, Map<String, Object> options, final Callable<Object> callable,
final OCommandOutputListener iListener) throws IOException {
if (!isClosed())
close();
OZIPCompressionUtil.uncompressDirectory(in, getStoragePath(), iListener);
}
@Override
protected void preOpenSteps() throws IOException {
if (configuration.binaryFormatVersion >= 11) {
if (dirtyFlag.exists())
dirtyFlag.open();
else {
dirtyFlag.create();
dirtyFlag.makeDirty();
}
} else {
if (dirtyFlag.exists())
dirtyFlag.open();
else {
dirtyFlag.create();
dirtyFlag.clearDirty();
}
}
}
@Override
protected void preCreateSteps() throws IOException {
dirtyFlag.create();
}
@Override
protected void postCloseSteps(boolean onDelete) throws IOException {
if (onDelete)
dirtyFlag.delete();
else {
dirtyFlag.clearDirty();
dirtyFlag.close();
}
}
@Override
protected void preCloseSteps() throws IOException {
try {
if (writeAheadLog != null) {
fuzzyCheckpointExecutor.shutdown();
if (!fuzzyCheckpointExecutor.awaitTermination(
OGlobalConfiguration.WAL_FUZZY_CHECKPOINT_SHUTDOWN_TIMEOUT.getValueAsInteger(), TimeUnit.SECONDS))
throw new OStorageException("Can not terminate fuzzy checkpoint task");
checkpointExecutor.shutdown();
if (!checkpointExecutor.awaitTermination(OGlobalConfiguration.WAL_FULL_CHECKPOINT_SHUTDOWN_TIMEOUT.getValueAsInteger(),
TimeUnit.SECONDS))
throw new OStorageException("Can not terminate full checkpoint task");
}
} catch (InterruptedException e) {
Thread.interrupted();
throw new OStorageException("Error on closing of storage '" + name, e);
}
}
@Override
protected void postDeleteSteps() {
File dbDir;// GET REAL DIRECTORY
dbDir = new File(OIOUtils.getPathFromDatabaseName(OSystemVariableResolver.resolveSystemVariables(url)));
if (!dbDir.exists() || !dbDir.isDirectory())
dbDir = dbDir.getParentFile();
// RETRIES
for (int i = 0; i < DELETE_MAX_RETRIES; ++i) {
if (dbDir != null && dbDir.exists() && dbDir.isDirectory()) {
int notDeletedFiles = 0;
// TRY TO DELETE ALL THE FILES
for (File f : dbDir.listFiles()) {
// DELETE ONLY THE SUPPORTED FILES
for (String ext : ALL_FILE_EXTENSIONS)
if (f.getPath().endsWith(ext)) {
if (!f.delete()) {
notDeletedFiles++;
}
break;
}
}
if (notDeletedFiles == 0) {
// TRY TO DELETE ALSO THE DIRECTORY IF IT'S EMPTY
dbDir.delete();
return;
}
} else
return;
OLogManager
.instance()
.debug(
this,
"Cannot delete database files because they are still locked by the OrientDB process: waiting %d ms and retrying %d/%d...",
DELETE_WAIT_TIME, i, DELETE_MAX_RETRIES);
}
throw new OStorageException("Cannot delete database '" + name + "' located in: " + dbDir + ". Database files seem locked");
}
protected void makeStorageDirty() throws IOException {
dirtyFlag.makeDirty();
}
protected void clearStorageDirty() throws IOException {
dirtyFlag.clearDirty();
}
@Override
protected boolean isDirty() throws IOException {
return dirtyFlag.isDirty();
}
protected void initWalAndDiskCache() throws IOException {
if (configuration.getContextConfiguration().getValueAsBoolean(OGlobalConfiguration.USE_WAL)) {
fuzzyCheckpointExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setDaemon(true);
return thread;
}
});
checkpointExecutor = Executors.newSingleThreadExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setDaemon(true);
return thread;
}
});
writeAheadLog = new ODiskWriteAheadLog(this);
final int fuzzyCheckpointDelay = OGlobalConfiguration.WAL_FUZZY_CHECKPOINT_INTERVAL.getValueAsInteger();
fuzzyCheckpointExecutor.scheduleWithFixedDelay(new Runnable() {
@Override
public void run() {
try {
makeFuzzyCheckPoint();
} catch (Throwable e) {
OLogManager.instance().error(this, "Error during background FUZZY checkpoint creation for storage " + name, e);
}
}
}, fuzzyCheckpointDelay, fuzzyCheckpointDelay, TimeUnit.SECONDS);
} else
writeAheadLog = null;
long diskCacheSize = OGlobalConfiguration.DISK_CACHE_SIZE.getValueAsLong() * 1024 * 1024;
long writeCacheSize = (long) Math.floor((((double) OGlobalConfiguration.DISK_WRITE_CACHE_PART.getValueAsInteger()) / 100.0)
* diskCacheSize);
long readCacheSize = diskCacheSize - writeCacheSize;
diskCache = new OReadWriteDiskCache(name, readCacheSize, writeCacheSize,
OGlobalConfiguration.DISK_CACHE_PAGE_SIZE.getValueAsInteger() * ONE_KB,
OGlobalConfiguration.DISK_WRITE_CACHE_PAGE_TTL.getValueAsLong() * 1000,
OGlobalConfiguration.DISK_WRITE_CACHE_PAGE_FLUSH_INTERVAL.getValueAsInteger(), this, writeAheadLog, false, true);
}
private boolean exists(String path) {
return new File(path + "/" + OMetadataDefault.CLUSTER_INTERNAL_NAME + OPaginatedCluster.DEF_EXTENSION).exists();
}
}