Examples of org.elasticsearch.index.store.Store

org.elasticsearch.index.store.Store
@author kimchy (shay.banon)

        NodeEnvironment nodeEnvironment = new NodeEnvironment(settings, environment);
        ByteBufferCache byteBufferCache = new ByteBufferCache(settings);


        ShardId shardId = new ShardId(new Index("index"), 1);
        String type = args.length > 0 ? args[0] : "ram";
        Store store;
        if (type.equalsIgnoreCase("ram")) {
            store = new RamStore(shardId, settings, null);
        } else if (type.equalsIgnoreCase("simple-fs")) {
            store = new SimpleFsStore(shardId, settings, new SimpleFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache);
        } else if (type.equalsIgnoreCase("mmap-fs")) {
            store = new NioFsStore(shardId, settings, new NioFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache);
        } else if (type.equalsIgnoreCase("nio-fs")) {
            store = new MmapFsStore(shardId, settings, new MmapFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache);
        } else if (type.equalsIgnoreCase("memory")) {
            store = new ByteBufferStore(shardId, settings, null, byteBufferCache);
        } else {
            throw new IllegalArgumentException("No type store [" + type + "]");
        }
        System.out.println("Using Store [" + store + "]");
        store.deleteContent();
        SimpleStoreBenchmark simpleStoreBenchmark = new SimpleStoreBenchmark(store)
                .numberStaticFiles(5).staticFileSize(new ByteSizeValue(5, ByteSizeUnit.MB))
                .dynamicFileSize(new ByteSizeValue(1, ByteSizeUnit.MB))
                .readerThreads(5).readerIterations(10)
                .writerThreads(2).writerIterations(10)
                .build();
        simpleStoreBenchmark.run();
        store.close();
    }

View Full Code Here

    public static void main(String[] args) throws Exception {
        ShardId shardId = new ShardId(new Index("index"), 1);
        Settings settings = EMPTY_SETTINGS;


//        Store store = new RamStore(shardId, settings);
        Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
//        Store store = new NioFsStore(shardId, settings);


        store.deleteContent();


        ThreadPool threadPool = new ThreadPool();
        SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
        Engine engine = new RobinEngine(shardId, settings, new ThreadPool(), new IndexSettingsService(shardId.index(), settings), store, deletionPolicy, new FsTranslog(shardId, EMPTY_SETTINGS, new File("work/fs-translog")), new LogByteSizeMergePolicyProvider(store, new IndexSettingsService(shardId.index(), EMPTY_SETTINGS)),
                new ConcurrentMergeSchedulerProvider(shardId, settings), new AnalysisService(shardId.index()), new SimilarityService(shardId.index()), new NoneBloomCache(shardId.index()));
        engine.start();


        SimpleEngineBenchmark benchmark = new SimpleEngineBenchmark(store, engine)
                .numberOfContentItems(1000)
                .searcherThreads(50).searcherIterations(10000)
                .writerThreads(10).writerIterations(10000)
                .refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
                .flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
                .create(false)
                .build();


        benchmark.run();


        engine.close();
        store.close();
        threadPool.shutdown();
    }

View Full Code Here


        // call this before we close the store, so we can release resources for it
        indicesLifecycle.afterIndexShardClosed(sId, delete);


        // if we delete or have no gateway or the store is not persistent, clean the store...
        Store store = shardInjector.getInstance(Store.class);
        if (delete || indexGateway.type().equals(NoneGateway.TYPE) || !indexStore.persistent()) {
            try {
                store.fullDelete();
            } catch (IOException e) {
                logger.warn("failed to clean store on shard deletion", e);
            }
        }
        // and close it
        try {
            store.close();
        } catch (IOException e) {
            logger.warn("failed to close store on shard deletion", e);
        }


        Injectors.close(injector);

View Full Code Here

    private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException {
        IndexService indexService = indicesService.indexService(shardId.index().name());
        if (indexService != null) {
            InternalIndexShard indexShard = (InternalIndexShard) indexService.shard(shardId.id());
            if (indexShard != null) {
                final Store store = indexShard.store();
                store.incRef();
                try {
                    return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty().asMap());
                } finally {
                    store.decRef();
                }
            }
        }
        // try and see if we an list unallocated
        IndexMetaData metaData = clusterService.state().metaData().index(shardId.index().name());

View Full Code Here

        return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), analyzer, source, mappingsModified);
    }


    protected Store createStore() throws IOException {
        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
        return new Store(shardId, EMPTY_SETTINGS, null, directoryService, new LeastUsedDistributor(directoryService));
    }

View Full Code Here

        return new Store(shardId, EMPTY_SETTINGS, null, directoryService, new LeastUsedDistributor(directoryService));
    }


    protected Store createStoreReplica() throws IOException {
        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
        return new Store(shardId, EMPTY_SETTINGS, null, directoryService, new LeastUsedDistributor(directoryService));
    }

View Full Code Here

        return ImmutableSettings.builder().put(AbstractMergePolicyProvider.INDEX_COMPOUND_FORMAT, value).build();
    }


    protected Store createStore(Settings settings) throws IOException {
        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
        return new Store(shardId, settings, null, directoryService, new LeastUsedDistributor(directoryService));
    }

View Full Code Here

        }


        // call this before we close the store, so we can release resources for it
        indicesLifecycle.afterIndexShardClosed(sId, indexShard);
        // if we delete or have no gateway or the store is not persistent, clean the store...
        Store store = shardInjector.getInstance(Store.class);
        // and close it
        try {
            store.close();
        } catch (Throwable e) {
            logger.warn("[{}] failed to close store on shard deletion", e, shardId);
        }
        Injectors.close(injector);

View Full Code Here

                final RecoveryStatus recoveryStatus = statusRef.status();
                // first, we go and move files that were created with the recovery id suffix to
                // the actual names, its ok if we have a corrupted index here, since we have replicas
                // to recover from in case of a full cluster shutdown just when this code executes...
                recoveryStatus.renameAllTempFiles();
                final Store store = recoveryStatus.store();
                // now write checksums
                recoveryStatus.legacyChecksums().write(store);


                for (String existingFile : store.directory().listAll()) {
                    // don't delete snapshot file, or the checksums file (note, this is extra protection since the Store won't delete checksum)
                    if (!request.snapshotFiles().contains(existingFile) && !Store.isChecksum(existingFile)) {
                        try {
                            store.directory().deleteFile(existingFile);
                        } catch (Exception e) {
                            // ignore, we don't really care, will get deleted later on
                        }
                    }
                }

View Full Code Here


        @Override
        public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                final Store store = recoveryStatus.store();
                IndexOutput indexOutput;
                if (request.position() == 0) {
                    indexOutput = recoveryStatus.openAndPutIndexOutput(request.name(), request.metadata(), store);
                } else {
                    indexOutput = recoveryStatus.getOpenIndexOutput(request.name());
                }
                if (recoverySettings.rateLimiter() != null) {
                    recoverySettings.rateLimiter().pause(request.content().length());
                }
                BytesReference content = request.content();
                if (!content.hasArray()) {
                    content = content.toBytesArray();
                }
                indexOutput.writeBytes(content.array(), content.arrayOffset(), content.length());
                recoveryStatus.state().getIndex().addRecoveredByteCount(content.length());
                RecoveryState.File file = recoveryStatus.state().getIndex().file(request.name());
                if (file != null) {
                    file.updateRecovered(request.length());
                }
                if (indexOutput.getFilePointer() >= request.length() || request.lastChunk()) {
                    try {
                        Store.verify(indexOutput);
                    } finally {
                        // we are done
                        indexOutput.close();
                    }
                    // write the checksum
                    recoveryStatus.legacyChecksums().add(request.metadata());
                    store.directory().sync(Collections.singleton(request.name()));
                    IndexOutput remove = recoveryStatus.removeOpenIndexOutputs(request.name());
                    recoveryStatus.state().getIndex().addRecoveredFileCount(1);
                    assert remove == null || remove == indexOutput; // remove maybe null if we got finished
                }
            }

View Full Code Here

0 1

TOP

Related Classes of org.elasticsearch.index.store.Store

org.apache.lucene.util.BytesRefBuilder

org.apache.lucene.util.Version

org.elasticsearch.benchmark.index.engine.SimpleEngineBenchmark

org.elasticsearch.benchmark.index.store.SimpleStoreBenchmark

org.elasticsearch.common.collect.Tuple

org.elasticsearch.common.lucene.store.InputStreamIndexInput

org.elasticsearch.index.engine.internal.InternalEngineTests

org.elasticsearch.index.merge.policy.MergePolicySettingsTest

org.elasticsearch.index.service.InternalIndexService

org.elasticsearch.index.shard.ShardId

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.