Package org.elasticsearch.common.unit

Examples of org.elasticsearch.common.unit.ByteSizeValue

72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    @Inject public IndexingMemoryBufferController(Settings settings, ThreadPool threadPool, IndicesService indicesService) {
        super(settings);
        this.threadPool = threadPool;
        this.indicesService = indicesService;

        ByteSizeValue indexingBuffer;
        String indexingBufferSetting = componentSettings.get("index_buffer_size", "10%");
        if (indexingBufferSetting.endsWith("%")) {
            double percent = Double.parseDouble(indexingBufferSetting.substring(0, indexingBufferSetting.length() - 1));
            indexingBuffer = new ByteSizeValue((long) (((double) JvmInfo.jvmInfo().mem().heapMax().bytes()) * (percent / 100)));
            ByteSizeValue minIndexingBuffer = componentSettings.getAsBytesSize("min_index_buffer_size", new ByteSizeValue(48, ByteSizeUnit.MB));
            ByteSizeValue maxIndexingBuffer = componentSettings.getAsBytesSize("max_index_buffer_size", null);

            if (indexingBuffer.bytes() < minIndexingBuffer.bytes()) {
                indexingBuffer = minIndexingBuffer;
            }
            if (maxIndexingBuffer != null && indexingBuffer.bytes() > maxIndexingBuffer.bytes()) {
                indexingBuffer = maxIndexingBuffer;
            }
        } else {
            indexingBuffer = ByteSizeValue.parseBytesSizeValue(indexingBufferSetting, null);
        }

        this.indexingBuffer = indexingBuffer;
        this.minShardIndexBufferSize = componentSettings.getAsBytesSize("min_shard_index_buffer_size", new ByteSizeValue(4, ByteSizeUnit.MB));
        // LUCENE MONITOR: Based on this thread, currently (based on Mike), having a large buffer does not make a lot of sense: https://issues.apache.org/jira/browse/LUCENE-2324?focusedCommentId=13005155&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-13005155
        this.maxShardIndexBufferSize = componentSettings.getAsBytesSize("max_shard_index_buffer_size", new ByteSizeValue(512, ByteSizeUnit.MB));

        this.inactiveTime = componentSettings.getAsTime("shard_inactive_time", TimeValue.timeValueMinutes(30));
        // we need to have this relatively small to move a shard from inactive to active fast (enough)
        this.interval = componentSettings.getAsTime("interval", TimeValue.timeValueSeconds(30));

View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
    private void calcAndSetShardIndexingBuffer(String reason) {
        int shardsCount = countShards();
        if (shardsCount == 0) {
            return;
        }
        ByteSizeValue shardIndexingBufferSize = calcShardIndexingBuffer(shardsCount);
        if (shardIndexingBufferSize == null) {
            return;
        }
        if (shardIndexingBufferSize.bytes() < minShardIndexBufferSize.bytes()) {
            shardIndexingBufferSize = minShardIndexBufferSize;
        }
        if (shardIndexingBufferSize.bytes() > maxShardIndexBufferSize.bytes()) {
            shardIndexingBufferSize = maxShardIndexBufferSize;
        }
        logger.debug("recalculating shard indexing buffer (reason={}), total is [{}] with [{}] active shards, each shard set to [{}]", reason, indexingBuffer, shardsCount, shardIndexingBufferSize);
        for (IndexService indexService : indicesService) {
            for (IndexShard indexShard : indexService) {
View Full Code Here
229
230
231
232
233
234
235
            }
        }
    }

    private ByteSizeValue calcShardIndexingBuffer(int shardsCount) {
        return new ByteSizeValue(indexingBuffer.bytes() / shardsCount);
    }
View Full Code Here
187
188
189
190
191
192
193
194
195
196
        }
        if (compress != null) {
            builder.field("compress", compress);
        }
        if (compressThreshold != -1) {
            builder.field("compress_threshold", new ByteSizeValue(compressThreshold).toString());
        }
        builder.endObject();
        return builder;
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
        super(store.shardId(), store.indexSettings());
        Preconditions.checkNotNull(store, "Store must be provided to merge policy");
        this.indexSettingsService = indexSettingsService;

        this.compoundFormat = indexSettings.getAsBoolean("index.compound_format", store.suggestUseCompoundFile());
        this.minMergeSize = componentSettings.getAsBytesSize("min_merge_size", new ByteSizeValue((long) (LogByteSizeMergePolicy.DEFAULT_MIN_MERGE_MB * 1024 * 1024), ByteSizeUnit.BYTES));
        this.maxMergeSize = componentSettings.getAsBytesSize("max_merge_size", new ByteSizeValue((long) LogByteSizeMergePolicy.DEFAULT_MAX_MERGE_MB, ByteSizeUnit.MB));
        this.mergeFactor = componentSettings.getAsInt("merge_factor", LogByteSizeMergePolicy.DEFAULT_MERGE_FACTOR);
        this.maxMergeDocs = componentSettings.getAsInt("max_merge_docs", LogByteSizeMergePolicy.DEFAULT_MAX_MERGE_DOCS);
        this.calibrateSizeByDeletes = componentSettings.getAsBoolean("calibrate_size_by_deletes", true);
        this.asyncMerge = indexSettings.getAsBoolean("index.merge.async", true);
        logger.debug("using [log_bytes_size] merge policy with merge_factor[{}], min_merge_size[{}], max_merge_size[{}], max_merge_docs[{}], calibrate_size_by_deletes[{}], async_merge[{}]",
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
        );
    }

    class ApplySettings implements IndexSettingsService.Listener {
        @Override public void onRefreshSettings(Settings settings) {
            ByteSizeValue minMergeSize = settings.getAsBytesSize("index.merge.policy.min_merge_size", LogByteSizeMergePolicyProvider.this.minMergeSize);
            if (!minMergeSize.equals(LogByteSizeMergePolicyProvider.this.minMergeSize)) {
                logger.info("updating min_merge_size from [{}] to [{}]", LogByteSizeMergePolicyProvider.this.minMergeSize, minMergeSize);
                LogByteSizeMergePolicyProvider.this.minMergeSize = minMergeSize;
                for (CustomLogByteSizeMergePolicy policy : policies) {
                    policy.setMinMergeMB(minMergeSize.mbFrac());
                }
            }

            ByteSizeValue maxMergeSize = settings.getAsBytesSize("index.merge.policy.max_merge_size", LogByteSizeMergePolicyProvider.this.maxMergeSize);
            if (!maxMergeSize.equals(LogByteSizeMergePolicyProvider.this.maxMergeSize)) {
                logger.info("updating max_merge_size from [{}] to [{}]", LogByteSizeMergePolicyProvider.this.maxMergeSize, maxMergeSize);
                LogByteSizeMergePolicyProvider.this.maxMergeSize = maxMergeSize;
                for (CustomLogByteSizeMergePolicy policy : policies) {
                    policy.setMaxMergeMB(maxMergeSize.mbFrac());
                }
            }

            int maxMergeDocs = settings.getAsInt("index.merge.policy.max_merge_docs", LogByteSizeMergePolicyProvider.this.maxMergeDocs);
            if (maxMergeDocs != LogByteSizeMergePolicyProvider.this.maxMergeDocs) {
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        this.indexSettingsService = indexSettingsService;

        this.compoundFormat = indexSettings.getAsBoolean("index.compound_format", store.suggestUseCompoundFile());
        this.asyncMerge = indexSettings.getAsBoolean("index.merge.async", true);
        this.expungeDeletesPctAllowed = componentSettings.getAsDouble("expunge_deletes_allowed", 10d); // percentage
        this.floorSegment = componentSettings.getAsBytesSize("floor_segment", new ByteSizeValue(2, ByteSizeUnit.MB));
        this.maxMergeAtOnce = componentSettings.getAsInt("max_merge_at_once", 10);
        this.maxMergeAtOnceExplicit = componentSettings.getAsInt("max_merge_at_once_explicit", 30);
        // TODO is this really a good default number for max_merge_segment, what happens for large indices, won't they end up with many segments?
        this.maxMergedSegment = componentSettings.getAsBytesSize("max_merge_segment", new ByteSizeValue(5, ByteSizeUnit.GB));
        this.segmentsPerTier = componentSettings.getAsDouble("segments_per_tier", 10d);
        this.reclaimDeletesWeight = componentSettings.getAsDouble("reclaim_deletes_weight", 2.0d);

        logger.debug("using [tiered] merge policy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merge_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}], async_merge[{}]",
                expungeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight, asyncMerge);
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
                for (CustomTieredMergePolicyProvider policy : policies) {
                    policy.setExpungeDeletesPctAllowed(expungeDeletesPctAllowed);
                }
            }

            ByteSizeValue floorSegment = settings.getAsBytesSize("index.merge.policy.floor_segment", TieredMergePolicyProvider.this.floorSegment);
            if (!floorSegment.equals(TieredMergePolicyProvider.this.floorSegment)) {
                logger.info("updating [floor_segment] from [{}] to [{}]", TieredMergePolicyProvider.this.floorSegment, floorSegment);
                TieredMergePolicyProvider.this.floorSegment = floorSegment;
                for (CustomTieredMergePolicyProvider policy : policies) {
                    policy.setFloorSegmentMB(floorSegment.mbFrac());
                }
            }

            int maxMergeAtOnce = settings.getAsInt("index.merge.policy.max_merge_at_once", TieredMergePolicyProvider.this.maxMergeAtOnce);
            if (maxMergeAtOnce != TieredMergePolicyProvider.this.maxMergeAtOnce) {
                logger.info("updating [max_merge_at_once] from [{}] to [{}]", TieredMergePolicyProvider.this.maxMergeAtOnce, maxMergeAtOnce);
                TieredMergePolicyProvider.this.maxMergeAtOnce = maxMergeAtOnce;
                for (CustomTieredMergePolicyProvider policy : policies) {
                    policy.setMaxMergeAtOnce(maxMergeAtOnce);
                }
            }

            int maxMergeAtOnceExplicit = settings.getAsInt("index.merge.policy.max_merge_at_once_explicit", TieredMergePolicyProvider.this.maxMergeAtOnceExplicit);
            if (maxMergeAtOnceExplicit != TieredMergePolicyProvider.this.maxMergeAtOnceExplicit) {
                logger.info("updating [max_merge_at_once_explicit] from [{}] to [{}]", TieredMergePolicyProvider.this.maxMergeAtOnceExplicit, maxMergeAtOnceExplicit);
                TieredMergePolicyProvider.this.maxMergeAtOnceExplicit = maxMergeAtOnceExplicit;
                for (CustomTieredMergePolicyProvider policy : policies) {
                    policy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
                }
            }

            ByteSizeValue maxMergedSegment = settings.getAsBytesSize("index.merge.policy.max_merged_segment", TieredMergePolicyProvider.this.maxMergedSegment);
            if (!maxMergedSegment.equals(TieredMergePolicyProvider.this.maxMergedSegment)) {
                logger.info("updating [max_merged_segment] from [{}] to [{}]", TieredMergePolicyProvider.this.maxMergedSegment, maxMergedSegment);
                TieredMergePolicyProvider.this.maxMergedSegment = maxMergedSegment;
                for (CustomTieredMergePolicyProvider policy : policies) {
                    policy.setFloorSegmentMB(maxMergedSegment.mbFrac());
                }
            }

            double segmentsPerTier = settings.getAsDouble("index.merge.policy.segments_per_tier", TieredMergePolicyProvider.this.segmentsPerTier);
            if (segmentsPerTier != TieredMergePolicyProvider.this.segmentsPerTier) {
View Full Code Here
170
171
172
173
174
175
176
        @Override public void writeTo(StreamOutput out) throws IOException {
            out.writeLong(total);
        }

        public ByteSizeValue total() {
            return new ByteSizeValue(total);
        }
View Full Code Here
202
203
204
205
206
207
208
        @Override public void writeTo(StreamOutput out) throws IOException {
            out.writeLong(total);
        }

        public ByteSizeValue total() {
            return new ByteSizeValue(total);
        }
View Full Code Here
TOP

Related Classes of org.elasticsearch.common.unit.ByteSizeValue

  • crate.elasticsearch.searchinto.BulkWriterCollector
  • io.crate.analyze.ExpressionToByteSizeValueVisitor
  • io.crate.analyze.SettingsAppliers$ByteSizeSettingsApplier
  • io.crate.http.netty.NettyHttpServerTransport
  • io.crate.operation.reference.sys.TestSysNodesExpressions$TestModule
  • org.apache.lucene.index.TrackingConcurrentMergeScheduler
  • org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes$JvmStats
  • org.elasticsearch.action.admin.indices.status.GatewayRecoveryStatus
  • org.elasticsearch.action.admin.indices.status.IndexShardStatus
  • org.elasticsearch.action.admin.indices.status.IndexStatus
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.