1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
22
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryUsage;
25 import java.util.concurrent.atomic.AtomicLong;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.ChoreService;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.ScheduledChore;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.io.hfile.BlockCache;
36 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
37 import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
38 import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
39 import org.apache.hadoop.util.ReflectionUtils;
40
41
42
43
44 @InterfaceAudience.Private
45 public class HeapMemoryManager {
46 private static final Log LOG = LogFactory.getLog(HeapMemoryManager.class);
47 private static final int CONVERT_TO_PERCENTAGE = 100;
48 private static final int CLUSTER_MINIMUM_MEMORY_THRESHOLD =
49 (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD);
50
51 public static final String BLOCK_CACHE_SIZE_MAX_RANGE_KEY = "hfile.block.cache.size.max.range";
52 public static final String BLOCK_CACHE_SIZE_MIN_RANGE_KEY = "hfile.block.cache.size.min.range";
53 public static final String MEMSTORE_SIZE_MAX_RANGE_KEY =
54 "hbase.regionserver.global.memstore.size.max.range";
55 public static final String MEMSTORE_SIZE_MIN_RANGE_KEY =
56 "hbase.regionserver.global.memstore.size.min.range";
57 public static final String HBASE_RS_HEAP_MEMORY_TUNER_PERIOD =
58 "hbase.regionserver.heapmemory.tuner.period";
59 public static final int HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD = 60 * 1000;
60 public static final String HBASE_RS_HEAP_MEMORY_TUNER_CLASS =
61 "hbase.regionserver.heapmemory.tuner.class";
62
63 public static final float HEAP_OCCUPANCY_ERROR_VALUE = -0.0f;
64
65 private float globalMemStorePercent;
66 private float globalMemStorePercentMinRange;
67 private float globalMemStorePercentMaxRange;
68
69 private float blockCachePercent;
70 private float blockCachePercentMinRange;
71 private float blockCachePercentMaxRange;
72 private float l2BlockCachePercent;
73
74 private float heapOccupancyPercent;
75
76 private final ResizableBlockCache blockCache;
77 private final FlushRequester memStoreFlusher;
78 private final Server server;
79 private final RegionServerAccounting regionServerAccounting;
80
81 private HeapMemoryTunerChore heapMemTunerChore = null;
82 private final boolean tunerOn;
83 private final int defaultChorePeriod;
84 private final float heapOccupancyLowWatermark;
85
86 private final long maxHeapSize;
87 {
88
89 long tempMaxHeap = -1L;
90 try {
91 final MemoryUsage usage = HeapMemorySizeUtil.safeGetHeapMemoryUsage();
92 if (usage != null) {
93 tempMaxHeap = usage.getMax();
94 }
95 } finally {
96 maxHeapSize = tempMaxHeap;
97 }
98 }
99
100 public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher,
101 Server server, RegionServerAccounting regionServerAccounting) {
102 BlockCache blockCache = CacheConfig.instantiateBlockCache(conf);
103 if (blockCache instanceof ResizableBlockCache) {
104 return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server,
105 regionServerAccounting);
106 }
107 return null;
108 }
109
110 HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher,
111 Server server, RegionServerAccounting regionServerAccounting) {
112 Configuration conf = server.getConfiguration();
113 this.blockCache = blockCache;
114 this.memStoreFlusher = memStoreFlusher;
115 this.server = server;
116 this.regionServerAccounting = regionServerAccounting;
117 this.tunerOn = doInit(conf);
118 this.defaultChorePeriod = conf.getInt(HBASE_RS_HEAP_MEMORY_TUNER_PERIOD,
119 HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD);
120 this.heapOccupancyLowWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_LOW_WATERMARK_KEY,
121 HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK);
122 }
123
124 private boolean doInit(Configuration conf) {
125 boolean tuningEnabled = true;
126 globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false);
127 blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
128 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
129 HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
130
131 globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
132 globalMemStorePercent);
133 globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
134 globalMemStorePercent);
135 if (globalMemStorePercent < globalMemStorePercentMinRange) {
136 LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
137 + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
138 + " because supplied value greater than initial memstore size value.");
139 globalMemStorePercentMinRange = globalMemStorePercent;
140 conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
141 }
142 if (globalMemStorePercent > globalMemStorePercentMaxRange) {
143 LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
144 + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
145 + " because supplied value less than initial memstore size value.");
146 globalMemStorePercentMaxRange = globalMemStorePercent;
147 conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
148 }
149 if (globalMemStorePercent == globalMemStorePercentMinRange
150 && globalMemStorePercent == globalMemStorePercentMaxRange) {
151 tuningEnabled = false;
152 }
153
154 blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent);
155 blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent);
156 if (blockCachePercent < blockCachePercentMinRange) {
157 LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " + blockCachePercent
158 + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
159 + " because supplied value greater than initial block cache size.");
160 blockCachePercentMinRange = blockCachePercent;
161 conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
162 }
163 if (blockCachePercent > blockCachePercentMaxRange) {
164 LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " + blockCachePercent
165 + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
166 + " because supplied value less than initial block cache size.");
167 blockCachePercentMaxRange = blockCachePercent;
168 conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
169 }
170 if (tuningEnabled && blockCachePercent == blockCachePercentMinRange
171 && blockCachePercent == blockCachePercentMaxRange) {
172 tuningEnabled = false;
173 }
174
175 int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
176 this.l2BlockCachePercent = HeapMemorySizeUtil.getL2BlockCacheHeapPercent(conf);
177 int bcul = (int) ((blockCachePercentMinRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
178 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
179 throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
180 + "the threshold required for successful cluster operation. "
181 + "The combined value cannot exceed 0.8. Please check the settings for "
182 + MEMSTORE_SIZE_MAX_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY
183 + " in your configuration. " + MEMSTORE_SIZE_MAX_RANGE_KEY + " is "
184 + globalMemStorePercentMaxRange + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " is "
185 + blockCachePercentMinRange);
186 }
187 gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
188 bcul = (int) ((blockCachePercentMaxRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
189 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
190 throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
191 + "the threshold required for successful cluster operation. "
192 + "The combined value cannot exceed 0.8. Please check the settings for "
193 + MEMSTORE_SIZE_MIN_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY
194 + " in your configuration. " + MEMSTORE_SIZE_MIN_RANGE_KEY + " is "
195 + globalMemStorePercentMinRange + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " is "
196 + blockCachePercentMaxRange);
197 }
198 return tuningEnabled;
199 }
200
201 public void start(ChoreService service) {
202 LOG.info("Starting HeapMemoryTuner chore.");
203 this.heapMemTunerChore = new HeapMemoryTunerChore();
204 service.scheduleChore(heapMemTunerChore);
205 if (tunerOn) {
206
207 memStoreFlusher.registerFlushRequestListener(heapMemTunerChore);
208 }
209 }
210
211 public void stop() {
212
213 LOG.info("Stopping HeapMemoryTuner chore.");
214 this.heapMemTunerChore.cancel(true);
215
216 }
217
218
219 boolean isTunerOn() {
220 return this.tunerOn;
221 }
222
223
224
225
226 public float getHeapOccupancyPercent() {
227 return this.heapOccupancyPercent == Float.MAX_VALUE ? HEAP_OCCUPANCY_ERROR_VALUE : this.heapOccupancyPercent;
228 }
229
230 private class HeapMemoryTunerChore extends ScheduledChore implements FlushRequestListener {
231 private HeapMemoryTuner heapMemTuner;
232 private AtomicLong blockedFlushCount = new AtomicLong();
233 private AtomicLong unblockedFlushCount = new AtomicLong();
234 private long evictCount = 0L;
235 private long cacheMissCount = 0L;
236 private TunerContext tunerContext = new TunerContext();
237 private boolean alarming = false;
238
239 public HeapMemoryTunerChore() {
240 super(server.getServerName() + "-HeapMemoryTunerChore", server, defaultChorePeriod);
241 Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass(
242 HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class);
243 heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration());
244 }
245
246 @Override
247 protected void chore() {
248
249 final MemoryUsage usage = HeapMemorySizeUtil.safeGetHeapMemoryUsage();
250 if (usage != null) {
251 heapOccupancyPercent = (float)usage.getUsed() / (float)usage.getCommitted();
252 } else {
253
254
255
256 heapOccupancyPercent = Float.MAX_VALUE;
257 }
258
259
260 if (heapOccupancyPercent >= heapOccupancyLowWatermark) {
261 if (!alarming) {
262 LOG.warn("heapOccupancyPercent " + heapOccupancyPercent +
263 " is above heap occupancy alarm watermark (" + heapOccupancyLowWatermark + ")");
264 alarming = true;
265 }
266
267 triggerNow();
268 try {
269
270
271 Thread.sleep(1000);
272 } catch (InterruptedException e) {
273
274 Thread.currentThread().interrupt();
275 }
276 } else {
277 if (alarming) {
278 LOG.info("heapOccupancyPercent " + heapOccupancyPercent +
279 " is now below the heap occupancy alarm watermark (" +
280 heapOccupancyLowWatermark + ")");
281 alarming = false;
282 }
283 }
284
285 if (tunerOn && !alarming) {
286 tune();
287 }
288 }
289
290 private void tune() {
291
292
293 long curEvictCount;
294 long curCacheMisCount;
295 curEvictCount = blockCache.getStats().getEvictedCount();
296 tunerContext.setEvictCount(curEvictCount - evictCount);
297 evictCount = curEvictCount;
298 curCacheMisCount = blockCache.getStats().getMissCachingCount();
299 tunerContext.setCacheMissCount(curCacheMisCount-cacheMissCount);
300 cacheMissCount = curCacheMisCount;
301 tunerContext.setBlockedFlushCount(blockedFlushCount.getAndSet(0));
302 tunerContext.setUnblockedFlushCount(unblockedFlushCount.getAndSet(0));
303 tunerContext.setCurBlockCacheUsed((float)blockCache.getCurrentSize() / maxHeapSize);
304 tunerContext.setCurMemStoreUsed(
305 (float)regionServerAccounting.getGlobalMemstoreSize() / maxHeapSize);
306 tunerContext.setCurBlockCacheSize(blockCachePercent);
307 tunerContext.setCurMemStoreSize(globalMemStorePercent);
308 TunerResult result = null;
309 try {
310 result = this.heapMemTuner.tune(tunerContext);
311 } catch (Throwable t) {
312 LOG.error("Exception thrown from the HeapMemoryTuner implementation", t);
313 }
314 if (result != null && result.needsTuning()) {
315 float memstoreSize = result.getMemstoreSize();
316 float blockCacheSize = result.getBlockCacheSize();
317 LOG.debug("From HeapMemoryTuner new memstoreSize: " + memstoreSize
318 + ". new blockCacheSize: " + blockCacheSize);
319 if (memstoreSize < globalMemStorePercentMinRange) {
320 LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is below min level "
321 + globalMemStorePercentMinRange + ". Resetting memstoreSize to min size");
322 memstoreSize = globalMemStorePercentMinRange;
323 } else if (memstoreSize > globalMemStorePercentMaxRange) {
324 LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is above max level "
325 + globalMemStorePercentMaxRange + ". Resetting memstoreSize to max size");
326 memstoreSize = globalMemStorePercentMaxRange;
327 }
328 if (blockCacheSize < blockCachePercentMinRange) {
329 LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
330 + " is below min level " + blockCachePercentMinRange
331 + ". Resetting blockCacheSize to min size");
332 blockCacheSize = blockCachePercentMinRange;
333 } else if (blockCacheSize > blockCachePercentMaxRange) {
334 LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
335 + " is above max level " + blockCachePercentMaxRange
336 + ". Resetting blockCacheSize to min size");
337 blockCacheSize = blockCachePercentMaxRange;
338 }
339 int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
340 int bcul = (int) ((blockCacheSize + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
341 if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
342 LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
343 + "the threshold required for successful cluster operation. "
344 + "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
345 + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
346 + blockCacheSize);
347
348 } else {
349 long newBlockCacheSize = (long) (maxHeapSize * blockCacheSize);
350 long newMemstoreSize = (long) (maxHeapSize * memstoreSize);
351 LOG.info("Setting block cache heap size to " + newBlockCacheSize
352 + " and memstore heap size to " + newMemstoreSize);
353 blockCachePercent = blockCacheSize;
354 blockCache.setMaxSize(newBlockCacheSize);
355 globalMemStorePercent = memstoreSize;
356 memStoreFlusher.setGlobalMemstoreLimit(newMemstoreSize);
357 }
358 } else if (LOG.isDebugEnabled()) {
359 LOG.debug("No changes made by HeapMemoryTuner.");
360 }
361 }
362
363 @Override
364 public void flushRequested(FlushType type, Region region) {
365 switch (type) {
366 case ABOVE_HIGHER_MARK:
367 blockedFlushCount.incrementAndGet();
368 break;
369 case ABOVE_LOWER_MARK:
370 unblockedFlushCount.incrementAndGet();
371 break;
372 default:
373
374 break;
375 }
376 }
377 }
378
379
380
381
382
383
384 public static final class TunerContext {
385 private long blockedFlushCount;
386 private long unblockedFlushCount;
387 private long evictCount;
388 private long cacheMissCount;
389 private float curBlockCacheUsed;
390 private float curMemStoreUsed;
391 private float curMemStoreSize;
392 private float curBlockCacheSize;
393
394 public long getBlockedFlushCount() {
395 return blockedFlushCount;
396 }
397
398 public void setBlockedFlushCount(long blockedFlushCount) {
399 this.blockedFlushCount = blockedFlushCount;
400 }
401
402 public long getUnblockedFlushCount() {
403 return unblockedFlushCount;
404 }
405
406 public void setUnblockedFlushCount(long unblockedFlushCount) {
407 this.unblockedFlushCount = unblockedFlushCount;
408 }
409
410 public long getEvictCount() {
411 return evictCount;
412 }
413
414 public void setEvictCount(long evictCount) {
415 this.evictCount = evictCount;
416 }
417
418 public float getCurMemStoreSize() {
419 return curMemStoreSize;
420 }
421
422 public void setCurMemStoreSize(float curMemStoreSize) {
423 this.curMemStoreSize = curMemStoreSize;
424 }
425
426 public float getCurBlockCacheSize() {
427 return curBlockCacheSize;
428 }
429
430 public void setCurBlockCacheSize(float curBlockCacheSize) {
431 this.curBlockCacheSize = curBlockCacheSize;
432 }
433
434 public long getCacheMissCount() {
435 return cacheMissCount;
436 }
437
438 public void setCacheMissCount(long cacheMissCount) {
439 this.cacheMissCount = cacheMissCount;
440 }
441
442 public float getCurBlockCacheUsed() {
443 return curBlockCacheUsed;
444 }
445
446 public void setCurBlockCacheUsed(float curBlockCacheUsed) {
447 this.curBlockCacheUsed = curBlockCacheUsed;
448 }
449
450 public float getCurMemStoreUsed() {
451 return curMemStoreUsed;
452 }
453
454 public void setCurMemStoreUsed(float d) {
455 this.curMemStoreUsed = d;
456 }
457 }
458
459
460
461
462
463 public static final class TunerResult {
464 private float memstoreSize;
465 private float blockCacheSize;
466 private final boolean needsTuning;
467
468 public TunerResult(boolean needsTuning) {
469 this.needsTuning = needsTuning;
470 }
471
472 public float getMemstoreSize() {
473 return memstoreSize;
474 }
475
476 public void setMemstoreSize(float memstoreSize) {
477 this.memstoreSize = memstoreSize;
478 }
479
480 public float getBlockCacheSize() {
481 return blockCacheSize;
482 }
483
484 public void setBlockCacheSize(float blockCacheSize) {
485 this.blockCacheSize = blockCacheSize;
486 }
487
488 public boolean needsTuning() {
489 return needsTuning;
490 }
491 }
492 }