1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile.bucket;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.FileNotFoundException;
27 import java.io.IOException;
28 import java.nio.ByteBuffer;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Random;
34 import java.util.Set;
35 import java.util.concurrent.locks.ReentrantReadWriteLock;
36
37 import com.google.common.collect.ImmutableMap;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseConfiguration;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
44 import org.apache.hadoop.hbase.io.hfile.BlockType;
45 import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
46 import org.apache.hadoop.hbase.io.hfile.CacheTestUtils.HFileBlockPair;
47 import org.apache.hadoop.hbase.io.hfile.Cacheable;
48 import org.apache.hadoop.hbase.io.hfile.HFileBlock;
49 import org.apache.hadoop.hbase.io.hfile.HFileContext;
50 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
51 import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.BucketSizeInfo;
52 import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.IndexStatistics;
53 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.RAMQueueEntry;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.junit.After;
56 import org.junit.Assert;
57 import org.junit.Before;
58 import org.junit.Test;
59 import org.junit.experimental.categories.Category;
60 import org.junit.runner.RunWith;
61 import org.junit.runners.Parameterized;
62 import org.mockito.Mockito;
63
64
65
66
67
68
69 @RunWith(Parameterized.class)
70 @Category(SmallTests.class)
71 public class TestBucketCache {
72
73 private static final Random RAND = new Random();
74
75 @Parameterized.Parameters(name = "{index}: blockSize={0}, bucketSizes={1}")
76 public static Iterable<Object[]> data() {
77 return Arrays.asList(new Object[][] {
78 { 8192, null },
79 {
80 16 * 1024,
81 new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024, 8 * 1024 + 1024, 16 * 1024 + 1024,
82 28 * 1024 + 1024, 32 * 1024 + 1024, 64 * 1024 + 1024, 96 * 1024 + 1024,
83 128 * 1024 + 1024 } } });
84 }
85
86 @Parameterized.Parameter(0)
87 public int constructedBlockSize;
88
89 @Parameterized.Parameter(1)
90 public int[] constructedBlockSizes;
91
92 BucketCache cache;
93 final int CACHE_SIZE = 1000000;
94 final int NUM_BLOCKS = 100;
95 final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS;
96 final int NUM_THREADS = 100;
97 final int NUM_QUERIES = 10000;
98
99 final long capacitySize = 32 * 1024 * 1024;
100 final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS;
101 final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS;
102 String ioEngineName = "heap";
103
104 private class MockedBucketCache extends BucketCache {
105
106 public MockedBucketCache(String ioEngineName, long capacity, int blockSize, int[] bucketSizes,
107 int writerThreads, int writerQLen, String persistencePath) throws FileNotFoundException,
108 IOException {
109 super(ioEngineName, capacity, blockSize, bucketSizes, writerThreads, writerQLen,
110 persistencePath);
111 super.wait_when_cache = true;
112 }
113
114 @Override
115 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory,
116 boolean cacheDataInL1) {
117 super.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
118 }
119
120 @Override
121 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
122 super.cacheBlock(cacheKey, buf);
123 }
124 }
125
126 @Before
127 public void setup() throws FileNotFoundException, IOException {
128 cache =
129 new MockedBucketCache(ioEngineName, capacitySize, constructedBlockSize,
130 constructedBlockSizes, writeThreads, writerQLen, null);
131 }
132
133 @After
134 public void tearDown() {
135 cache.shutdown();
136 }
137
138
139
140
141 private static <T> T randFrom(List<T> a) {
142 return a.get(RAND.nextInt(a.size()));
143 }
144
145 @Test
146 public void testBucketAllocator() throws BucketAllocatorException {
147 BucketAllocator mAllocator = cache.getAllocator();
148
149
150
151 final List<Integer> BLOCKSIZES = Arrays.asList(4 * 1024, 8 * 1024, 64 * 1024, 96 * 1024);
152
153 boolean full = false;
154 ArrayList<Long> allocations = new ArrayList<Long>();
155
156
157 List<Integer> tmp = new ArrayList<Integer>(BLOCKSIZES);
158 while (!full) {
159 Integer blockSize = null;
160 try {
161 blockSize = randFrom(tmp);
162 allocations.add(mAllocator.allocateBlock(blockSize));
163 } catch (CacheFullException cfe) {
164 tmp.remove(blockSize);
165 if (tmp.isEmpty()) full = true;
166 }
167 }
168
169 for (Integer blockSize : BLOCKSIZES) {
170 BucketSizeInfo bucketSizeInfo = mAllocator.roundUpToBucketSizeInfo(blockSize);
171 IndexStatistics indexStatistics = bucketSizeInfo.statistics();
172 assertEquals("unexpected freeCount for " + bucketSizeInfo, 0, indexStatistics.freeCount());
173 }
174
175 for (long offset : allocations) {
176 assertEquals(mAllocator.sizeOfAllocation(offset), mAllocator.freeBlock(offset));
177 }
178 assertEquals(0, mAllocator.getUsedSize());
179 }
180
181 @Test
182 public void testCacheSimple() throws Exception {
183 CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES);
184 }
185
186 @Test
187 public void testCacheMultiThreadedSingleKey() throws Exception {
188 CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, 2 * NUM_THREADS, 2 * NUM_QUERIES);
189 }
190
191 @Test
192 public void testHeapSizeChanges() throws Exception {
193 cache.stopWriterThreads();
194 CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
195 }
196
197 private void waitUntilFlushedToBucket(BucketCache cache, BlockCacheKey cacheKey)
198 throws InterruptedException {
199 while (!cache.backingMap.containsKey(cacheKey) || cache.ramCache.containsKey(cacheKey)) {
200 Thread.sleep(100);
201 }
202 }
203
204
205
206 private void cacheAndWaitUntilFlushedToBucket(BucketCache cache, BlockCacheKey cacheKey,
207 Cacheable block) throws InterruptedException {
208 cache.cacheBlock(cacheKey, block);
209 waitUntilFlushedToBucket(cache, cacheKey);
210 }
211
212 @Test
213 public void testMemoryLeak() throws Exception {
214 final BlockCacheKey cacheKey = new BlockCacheKey("dummy", 1L);
215 cacheAndWaitUntilFlushedToBucket(cache, cacheKey, new CacheTestUtils.ByteArrayCacheable(
216 new byte[10]));
217 long lockId = cache.backingMap.get(cacheKey).offset();
218 ReentrantReadWriteLock lock = cache.offsetLock.getLock(lockId);
219 lock.writeLock().lock();
220 Thread evictThread = new Thread("evict-block") {
221
222 @Override
223 public void run() {
224 cache.evictBlock(cacheKey);
225 }
226
227 };
228 evictThread.start();
229 cache.offsetLock.waitForWaiters(lockId, 1);
230 cache.blockEvicted(cacheKey, cache.backingMap.remove(cacheKey), true);
231 cacheAndWaitUntilFlushedToBucket(cache, cacheKey, new CacheTestUtils.ByteArrayCacheable(
232 new byte[10]));
233 lock.writeLock().unlock();
234 evictThread.join();
235 assertEquals(1L, cache.getBlockCount());
236 assertTrue(cache.getCurrentSize() > 0L);
237 assertTrue("We should have a block!", cache.iterator().hasNext());
238 }
239
240 @Test
241 public void testRetrieveFromFile() throws Exception {
242 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
243 Path testDir = TEST_UTIL.getDataTestDir();
244 TEST_UTIL.getTestFileSystem().mkdirs(testDir);
245
246 BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
247 constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir
248 + "/bucket.persistence");
249 long usedSize = bucketCache.getAllocator().getUsedSize();
250 assertTrue(usedSize == 0);
251
252 HFileBlockPair[] blocks = CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
253
254 for (HFileBlockPair block : blocks) {
255 bucketCache.cacheBlock(block.getBlockName(), block.getBlock());
256 }
257 for (HFileBlockPair block : blocks) {
258 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
259 }
260 usedSize = bucketCache.getAllocator().getUsedSize();
261 assertTrue(usedSize != 0);
262
263 bucketCache.shutdown();
264
265
266 bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
267 constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir
268 + "/bucket.persistence");
269 assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
270
271 bucketCache.shutdown();
272
273
274
275 int[] smallBucketSizes = new int[] { 2 * 1024 + 1024, 4 * 1024 + 1024 };
276 bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize,
277 constructedBlockSize, smallBucketSizes, writeThreads,
278 writerQLen, testDir + "/bucket.persistence");
279 assertEquals(0, bucketCache.getAllocator().getUsedSize());
280 assertEquals(0, bucketCache.backingMap.size());
281
282 TEST_UTIL.cleanupTestDir();
283 }
284
285 @Test
286 public void testGetPartitionSize() throws IOException {
287
288 validateGetPartitionSize(cache, BucketCache.DEFAULT_SINGLE_FACTOR, BucketCache.DEFAULT_MIN_FACTOR);
289
290 Configuration conf = HBaseConfiguration.create();
291 conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f);
292 conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f);
293 conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f);
294 conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f);
295
296 BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize,
297 constructedBlockSizes, writeThreads, writerQLen, null, 100, conf);
298
299 validateGetPartitionSize(cache, 0.1f, 0.5f);
300 validateGetPartitionSize(cache, 0.7f, 0.5f);
301 validateGetPartitionSize(cache, 0.2f, 0.5f);
302 }
303
304 @Test
305 public void testValidBucketCacheConfigs() throws IOException {
306 Configuration conf = HBaseConfiguration.create();
307 conf.setFloat(BucketCache.ACCEPT_FACTOR_CONFIG_NAME, 0.9f);
308 conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f);
309 conf.setFloat(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME, 0.5f);
310 conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f);
311 conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f);
312 conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f);
313
314 BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize,
315 constructedBlockSizes, writeThreads, writerQLen, null, 100, conf);
316
317 assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getAcceptableFactor(), 0.9f, 0);
318 assertEquals(BucketCache.MIN_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMinFactor(), 0.5f, 0);
319 assertEquals(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getExtraFreeFactor(), 0.5f, 0);
320 assertEquals(BucketCache.SINGLE_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getSingleFactor(), 0.1f, 0);
321 assertEquals(BucketCache.MULTI_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMultiFactor(), 0.7f, 0);
322 assertEquals(BucketCache.MEMORY_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMemoryFactor(), 0.2f, 0);
323 }
324
325 @Test
326 public void testInvalidAcceptFactorConfig() throws IOException {
327 float[] configValues = {-1f, 0.2f, 0.86f, 1.05f};
328 boolean[] expectedOutcomes = {false, false, true, false};
329 Map<String, float[]> configMappings = ImmutableMap.of(BucketCache.ACCEPT_FACTOR_CONFIG_NAME, configValues);
330 Configuration conf = HBaseConfiguration.create();
331 checkConfigValues(conf, configMappings, expectedOutcomes);
332 }
333
334 @Test
335 public void testInvalidMinFactorConfig() throws IOException {
336 float[] configValues = {-1f, 0f, 0.96f, 1.05f};
337
338 boolean[] expectedOutcomes = {false, true, false, false};
339 Map<String, float[]> configMappings = ImmutableMap.of(BucketCache.MIN_FACTOR_CONFIG_NAME, configValues);
340 Configuration conf = HBaseConfiguration.create();
341 checkConfigValues(conf, configMappings, expectedOutcomes);
342 }
343
344 @Test
345 public void testInvalidExtraFreeFactorConfig() throws IOException {
346 float[] configValues = {-1f, 0f, 0.2f, 1.05f};
347
348 boolean[] expectedOutcomes = {false, true, true, true};
349 Map<String, float[]> configMappings = ImmutableMap.of(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME, configValues);
350 Configuration conf = HBaseConfiguration.create();
351 checkConfigValues(conf, configMappings, expectedOutcomes);
352 }
353
354 @Test
355 public void testInvalidCacheSplitFactorConfig() throws IOException {
356 float[] singleFactorConfigValues = {0.2f, 0f, -0.2f, 1f};
357 float[] multiFactorConfigValues = {0.4f, 0f, 1f, .05f};
358 float[] memoryFactorConfigValues = {0.4f, 0f, 0.2f, .5f};
359
360 boolean[] expectedOutcomes = {true, false, false, false};
361 Map<String, float[]> configMappings = ImmutableMap.of(BucketCache.SINGLE_FACTOR_CONFIG_NAME,
362 singleFactorConfigValues, BucketCache.MULTI_FACTOR_CONFIG_NAME, multiFactorConfigValues,
363 BucketCache.MEMORY_FACTOR_CONFIG_NAME, memoryFactorConfigValues);
364 Configuration conf = HBaseConfiguration.create();
365 checkConfigValues(conf, configMappings, expectedOutcomes);
366 }
367
368 private void checkConfigValues(Configuration conf, Map<String, float[]> configMap, boolean[] expectSuccess) throws IOException {
369 Set<String> configNames = configMap.keySet();
370 for (int i = 0; i < expectSuccess.length; i++) {
371 try {
372 for (String configName : configNames) {
373 conf.setFloat(configName, configMap.get(configName)[i]);
374 }
375 new BucketCache(ioEngineName, capacitySize, constructedBlockSize,
376 constructedBlockSizes, writeThreads, writerQLen, null, 100, conf);
377 assertTrue("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]);
378 } catch (IllegalArgumentException e) {
379 assertFalse("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]);
380 }
381 }
382 }
383
384 private void validateGetPartitionSize(BucketCache bucketCache, float partitionFactor, float minFactor) {
385 long expectedOutput = (long) Math.floor(bucketCache.getAllocator().getTotalSize() * partitionFactor * minFactor);
386 assertEquals(expectedOutput, bucketCache.getPartitionSize(partitionFactor));
387 }
388
389 @Test
390 public void testOffsetProducesPositiveOutput() {
391
392
393 long testValue = 549888460800L;
394 BucketCache.BucketEntry bucketEntry = new BucketCache.BucketEntry(testValue, 10, 10L, true);
395 assertEquals(testValue, bucketEntry.offset());
396 }
397
398 @Test
399 public void testCacheBlockNextBlockMetadataMissing() throws Exception {
400 int size = 100;
401 int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
402 byte[] byteArr = new byte[length];
403 ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
404 HFileContext meta = new HFileContextBuilder().build();
405 HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf,
406 HFileBlock.FILL_HEADER, -1, 52, -1, meta);
407 HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf,
408 HFileBlock.FILL_HEADER, -1, -1, -1, meta);
409
410 BlockCacheKey key = new BlockCacheKey("key1", 0);
411 ByteBuffer actualBuffer = ByteBuffer.allocate(length);
412 ByteBuffer block1Buffer = ByteBuffer.allocate(length);
413 ByteBuffer block2Buffer = ByteBuffer.allocate(length);
414 blockWithNextBlockMetadata.serialize(block1Buffer, true);
415 blockWithoutNextBlockMetadata.serialize(block2Buffer, true);
416
417
418 CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata,
419 actualBuffer, block1Buffer);
420
421 waitUntilFlushedToBucket(cache, key);
422
423 CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata,
424 actualBuffer, block1Buffer);
425
426
427 cache.evictBlock(key);
428 assertNull(cache.getBlock(key, false, false, false));
429 CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata,
430 actualBuffer, block2Buffer);
431
432 waitUntilFlushedToBucket(cache, key);
433
434 CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata,
435 actualBuffer, block1Buffer);
436 }
437
438 @Test
439 public void testFreeBlockWhenIOEngineWriteFailure() throws IOException {
440
441 int size = 100, offset = 20;
442 int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
443 ByteBuffer buf = ByteBuffer.allocate(length);
444 HFileContext meta = new HFileContextBuilder().build();
445 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER,
446 offset, 52, -1, meta);
447
448
449 IOEngine ioEngine = Mockito.mock(IOEngine.class);
450
451 Mockito.doThrow(RuntimeException.class).when(ioEngine).write(Mockito.any(ByteBuffer.class),
452 Mockito.anyLong());
453
454
455 long availableSpace = 1024 * 1024 * 1024L;
456 BucketAllocator allocator = new BucketAllocator(availableSpace, null);
457
458 BlockCacheKey key = new BlockCacheKey("dummy", 1L);
459 RAMQueueEntry re = new RAMQueueEntry(key, block, 1, true);
460
461 Assert.assertEquals(0, allocator.getUsedSize());
462 try {
463 re.writeToCache(ioEngine, allocator, new UniqueIndexMap<Integer>(), null);
464 Assert.fail();
465 } catch (Exception e) {
466 }
467 Assert.assertEquals(0, allocator.getUsedSize());
468 }
469 }