1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.hfile;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.io.DataOutputStream;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.List;
28
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.KeyValue;
31 import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
32 import org.apache.hadoop.hbase.testclassification.SmallTests;
33 import org.apache.hadoop.hbase.io.HeapSize;
34 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
35 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
36 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
37 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
38 import org.apache.hadoop.hbase.util.ChecksumType;
39 import org.apache.hadoop.hbase.util.RedundantKVGenerator;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42 import org.junit.runner.RunWith;
43 import org.junit.runners.Parameterized;
44 import org.junit.runners.Parameterized.Parameters;
45
46 @RunWith(Parameterized.class)
47 @Category(SmallTests.class)
48 public class TestHFileDataBlockEncoder {
49 private HFileDataBlockEncoder blockEncoder;
50 private RedundantKVGenerator generator = new RedundantKVGenerator();
51 private boolean includesMemstoreTS;
52
53
54
55
56
57 public TestHFileDataBlockEncoder(HFileDataBlockEncoder blockEncoder,
58 boolean includesMemstoreTS) {
59 this.blockEncoder = blockEncoder;
60 this.includesMemstoreTS = includesMemstoreTS;
61 System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding()
62 + ", includesMemstoreTS: " + includesMemstoreTS);
63 }
64
65
66
67
68
69 @Test
70 public void testEncodingWithCache() throws IOException {
71 testEncodingWithCacheInternals(false);
72 testEncodingWithCacheInternals(true);
73 }
74
75 private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
76 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
77 HFileBlock block = getSampleHFileBlock(kvs, useTag);
78 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag);
79
80 LruBlockCache blockCache =
81 new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
82 BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
83 blockCache.cacheBlock(cacheKey, cacheBlock);
84
85 HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
86 assertTrue(heapSize instanceof HFileBlock);
87
88 HFileBlock returnedBlock = (HFileBlock) heapSize;;
89
90 if (blockEncoder.getDataBlockEncoding() ==
91 DataBlockEncoding.NONE) {
92 assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly());
93 } else {
94 if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
95 System.out.println(blockEncoder);
96 }
97 assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
98 }
99 }
100
101
102 @Test
103 public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
104 testHeaderSizeInCacheWithoutChecksumInternals(false);
105 testHeaderSizeInCacheWithoutChecksumInternals(true);
106 }
107
108 private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException {
109 int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
110
111 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags);
112 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
113 int size = keyValues.limit();
114 ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
115 buf.position(headerSize);
116 keyValues.rewind();
117 buf.put(keyValues);
118 HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false)
119 .withIncludesMvcc(includesMemstoreTS)
120 .withIncludesTags(useTags)
121 .withBlockSize(0)
122 .withChecksumType(ChecksumType.NULL)
123 .build();
124 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
125 HFileBlock.FILL_HEADER, 0,
126 0, -1, hfileContext);
127 HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags);
128 assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
129 }
130
131
132
133
134
135 @Test
136 public void testEncoding() throws IOException {
137 testEncodingInternals(false);
138 testEncodingInternals(true);
139 }
140
141 private void testEncodingInternals(boolean useTag) throws IOException {
142
143 List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
144 HFileBlock block = getSampleHFileBlock(kvs, useTag);
145 HFileBlock blockOnDisk = createBlockOnDisk(kvs, block, useTag);
146
147 if (blockEncoder.getDataBlockEncoding() !=
148 DataBlockEncoding.NONE) {
149 assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
150 assertEquals(blockEncoder.getDataBlockEncoding().getId(),
151 blockOnDisk.getDataBlockEncodingId());
152 } else {
153 assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
154 }
155 }
156
157 private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) {
158 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS);
159 int size = keyValues.limit();
160 ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
161 buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
162 keyValues.rewind();
163 buf.put(keyValues);
164 HFileContext meta = new HFileContextBuilder()
165 .withIncludesMvcc(includesMemstoreTS)
166 .withIncludesTags(useTag)
167 .withHBaseCheckSum(true)
168 .withCompression(Algorithm.NONE)
169 .withBlockSize(0)
170 .withChecksumType(ChecksumType.NULL)
171 .build();
172 HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
173 HFileBlock.FILL_HEADER, 0,
174 0, -1, meta);
175 return b;
176 }
177
178 private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
179 throws IOException {
180 int size;
181 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
182 blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
183 block.getHFileContext());
184
185 ByteArrayOutputStream baos = new ByteArrayOutputStream();
186 baos.write(block.getDummyHeaderForVersion());
187 DataOutputStream dos = new DataOutputStream(baos);
188 blockEncoder.startBlockEncoding(context, dos);
189 for (KeyValue kv : kvs) {
190 blockEncoder.encode(kv, context, dos);
191 }
192 blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA);
193 byte[] encodedBytes = baos.toByteArray();
194 size = encodedBytes.length - block.getDummyHeaderForVersion().length;
195 return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
196 HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1,
197 block.getHFileContext());
198 }
199
200
201
202
203 @Parameters
204 public static Collection<Object[]> getAllConfigurations() {
205 List<Object[]> configurations =
206 new ArrayList<Object[]>();
207
208 for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
209 for (boolean includesMemstoreTS : new boolean[] { false, true }) {
210 HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ?
211 NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo);
212 configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) });
213 }
214 }
215
216 return configurations;
217 }
218 }