1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile.bucket;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.ObjectOutputStream;
29 import java.io.OutputStreamWriter;
30 import java.util.concurrent.ConcurrentMap;
31
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
35 import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
36 import org.apache.hadoop.hbase.io.hfile.Cacheable;
37 import org.apache.hadoop.hbase.testclassification.SmallTests;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40
41
42
43
44 @Category(SmallTests.class)
45 public class TestVerifyBucketCacheFile {
46 final int constructedBlockSize = 8 * 1024;
47 final long capacitySize = 32 * 1024 * 1024;
48 final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS;
49 final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS;
50
51
52
53
54
55
56
57
58
59
60
61
62
63 @Test
64 public void testRetrieveFromFile() throws Exception {
65 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 Path testDir = TEST_UTIL.getDataTestDir();
67 TEST_UTIL.getTestFileSystem().mkdirs(testDir);
68
69 BucketCache bucketCache =
70 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
71 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
72 long usedSize = bucketCache.getAllocator().getUsedSize();
73 assertTrue(usedSize == 0);
74 CacheTestUtils.HFileBlockPair[] blocks =
75 CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
76
77 for (CacheTestUtils.HFileBlockPair block : blocks) {
78 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
79 }
80 usedSize = bucketCache.getAllocator().getUsedSize();
81 assertTrue(usedSize != 0);
82
83 bucketCache.shutdown();
84
85 bucketCache =
86 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
87 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
88 assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
89
90 bucketCache.shutdown();
91
92
93 File cacheFile = new File(testDir + "/bucket.cache");
94 assertTrue(cacheFile.delete());
95
96 bucketCache =
97 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
98 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
99 assertEquals(0, bucketCache.getAllocator().getUsedSize());
100 assertEquals(0, bucketCache.backingMap.size());
101
102 for (CacheTestUtils.HFileBlockPair block : blocks) {
103 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
104 }
105 usedSize = bucketCache.getAllocator().getUsedSize();
106 assertTrue(usedSize != 0);
107
108 bucketCache.shutdown();
109
110
111 File mapFile = new File(testDir + "/bucket.persistence");
112 assertTrue(mapFile.delete());
113
114 bucketCache =
115 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
116 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
117 assertEquals(0, bucketCache.getAllocator().getUsedSize());
118 assertEquals(0, bucketCache.backingMap.size());
119
120 TEST_UTIL.cleanupTestDir();
121 }
122
123
124
125
126
127
128
129
130 @Test
131 public void testModifiedBucketCacheFileData() throws Exception {
132 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
133 Path testDir = TEST_UTIL.getDataTestDir();
134 TEST_UTIL.getTestFileSystem().mkdirs(testDir);
135
136 BucketCache bucketCache =
137 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
138 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
139 long usedSize = bucketCache.getAllocator().getUsedSize();
140 assertTrue(usedSize == 0);
141
142 CacheTestUtils.HFileBlockPair[] blocks =
143 CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
144
145 for (CacheTestUtils.HFileBlockPair block : blocks) {
146 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
147 }
148 usedSize = bucketCache.getAllocator().getUsedSize();
149 assertTrue(usedSize != 0);
150
151 bucketCache.shutdown();
152
153
154 String file = testDir + "/bucket.cache";
155 try(BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
156 new FileOutputStream(file, false)))) {
157 out.write("test bucket cache");
158 }
159
160
161 bucketCache =
162 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
163 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
164 assertEquals(0, bucketCache.getAllocator().getUsedSize());
165 assertEquals(0, bucketCache.backingMap.size());
166
167 TEST_UTIL.cleanupTestDir();
168 }
169
170
171
172
173
174
175
176
177
178 @Test
179 public void testModifiedBucketCacheFileTime() throws Exception {
180 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
181 Path testDir = TEST_UTIL.getDataTestDir();
182 TEST_UTIL.getTestFileSystem().mkdirs(testDir);
183
184 BucketCache bucketCache =
185 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
186 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
187 long usedSize = bucketCache.getAllocator().getUsedSize();
188 assertTrue(usedSize == 0);
189
190 CacheTestUtils.HFileBlockPair[] blocks =
191 CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
192
193 for (CacheTestUtils.HFileBlockPair block : blocks) {
194 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
195 }
196 usedSize = bucketCache.getAllocator().getUsedSize();
197 assertTrue(usedSize != 0);
198
199 bucketCache.shutdown();
200
201
202 File file = new File(testDir + "/bucket.cache");
203 assertTrue(file.setLastModified(System.currentTimeMillis() + 1000));
204
205
206 bucketCache =
207 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
208 null, writeThreads, writerQLen, testDir + "/bucket.persistence");
209 assertEquals(0, bucketCache.getAllocator().getUsedSize());
210 assertEquals(0, bucketCache.backingMap.size());
211
212 TEST_UTIL.cleanupTestDir();
213 }
214
215
216
217
218
219
220
221
222 @Test
223 public void compatibilityTest() throws Exception {
224 HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
225 Path testDir = TEST_UTIL.getDataTestDir();
226 TEST_UTIL.getTestFileSystem().mkdirs(testDir);
227 String persistencePath = testDir + "/bucket.persistence";
228 BucketCache bucketCache =
229 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
230 null, writeThreads, writerQLen, persistencePath);
231 long usedSize = bucketCache.getAllocator().getUsedSize();
232 assertTrue(usedSize == 0);
233
234 CacheTestUtils.HFileBlockPair[] blocks =
235 CacheTestUtils.generateHFileBlocks(constructedBlockSize, 1);
236
237 for (CacheTestUtils.HFileBlockPair block : blocks) {
238 cacheAndWaitUntilFlushedToBucket(bucketCache, block.getBlockName(), block.getBlock());
239 }
240 usedSize = bucketCache.getAllocator().getUsedSize();
241 assertTrue(usedSize != 0);
242
243 persistToFileInOldWay(persistencePath + ".old", bucketCache.getMaxSize(),
244 bucketCache.backingMap, bucketCache.getDeserialiserMap());
245 bucketCache.shutdown();
246
247
248 bucketCache =
249 new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize,
250 null, writeThreads, writerQLen, persistencePath + ".old");
251 assertEquals(usedSize, bucketCache.getAllocator().getUsedSize());
252 assertEquals(blocks.length, bucketCache.backingMap.size());
253 }
254
255 private void persistToFileInOldWay(String persistencePath, long cacheCapacity,
256 ConcurrentMap backingMap, UniqueIndexMap deserialiserMap)
257 throws IOException {
258 try(ObjectOutputStream oos = new ObjectOutputStream(
259 new FileOutputStream(persistencePath, false))) {
260 oos.writeLong(cacheCapacity);
261 oos.writeUTF(FileIOEngine.class.getName());
262 oos.writeUTF(backingMap.getClass().getName());
263 oos.writeObject(deserialiserMap);
264 oos.writeObject(backingMap);
265 }
266 }
267
268 private void waitUntilFlushedToBucket(BucketCache cache, BlockCacheKey cacheKey)
269 throws InterruptedException {
270 while (!cache.backingMap.containsKey(cacheKey) || cache.ramCache.containsKey(cacheKey)) {
271 Thread.sleep(100);
272 }
273 }
274
275
276
277 private void cacheAndWaitUntilFlushedToBucket(BucketCache cache, BlockCacheKey cacheKey,
278 Cacheable block) throws InterruptedException {
279 cache.cacheBlock(cacheKey, block);
280 waitUntilFlushedToBucket(cache, cacheKey);
281 }
282 }