1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.ipc;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.Arrays;
25
26 import org.apache.commons.lang.time.StopWatch;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.commons.logging.impl.Log4JLogger;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellScanner;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HBaseConfiguration;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.codec.Codec;
36 import org.apache.hadoop.hbase.codec.KeyValueCodec;
37 import org.apache.hadoop.hbase.io.SizedCellScanner;
38 import org.apache.hadoop.hbase.testclassification.ClientTests;
39 import org.apache.hadoop.hbase.testclassification.SmallTests;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.ClassSize;
42 import org.apache.hadoop.io.compress.CompressionCodec;
43 import org.apache.hadoop.io.compress.DefaultCodec;
44 import org.apache.hadoop.io.compress.GzipCodec;
45 import org.apache.log4j.Level;
46 import org.junit.Before;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49
50 @Category({ ClientTests.class, SmallTests.class })
51 public class TestCellBlockBuilder {
52 private static final Log LOG = LogFactory.getLog(TestCellBlockBuilder.class);
53
54 private CellBlockBuilder builder;
55
56 @Before
57 public void before() {
58 this.builder = new CellBlockBuilder(HBaseConfiguration.create());
59 }
60
61 @Test
62 public void testBuildCellBlock() throws IOException {
63 doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), null);
64 doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new DefaultCodec());
65 doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new GzipCodec());
66 }
67
68 static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
69 final CompressionCodec compressor) throws IOException {
70 doBuildCellBlockUndoCellBlock(builder, codec, compressor, 10, 1, false);
71 }
72
73 static void doBuildCellBlockUndoCellBlock(final CellBlockBuilder builder, final Codec codec,
74 final CompressionCodec compressor, final int count, final int size, final boolean sized)
75 throws IOException {
76 Cell[] cells = getCells(count, size);
77 CellScanner cellScanner = sized ? getSizedCellScanner(cells)
78 : CellUtil.createCellScanner(Arrays.asList(cells).iterator());
79 ByteBuffer bb = builder.buildCellBlock(codec, compressor, cellScanner);
80 cellScanner = builder.createCellScanner(codec, compressor, bb);
81 int i = 0;
82 while (cellScanner.advance()) {
83 i++;
84 }
85 assertEquals(count, i);
86 }
87
88 static CellScanner getSizedCellScanner(final Cell[] cells) {
89 int size = -1;
90 for (Cell cell : cells) {
91 size += CellUtil.estimatedSerializedSizeOf(cell);
92 }
93 final int totalSize = ClassSize.align(size);
94 final CellScanner cellScanner = CellUtil.createCellScanner(cells);
95 return new SizedCellScanner() {
96 @Override
97 public long heapSize() {
98 return totalSize;
99 }
100
101 @Override
102 public Cell current() {
103 return cellScanner.current();
104 }
105
106 @Override
107 public boolean advance() throws IOException {
108 return cellScanner.advance();
109 }
110 };
111 }
112
113 static Cell[] getCells(final int howMany) {
114 return getCells(howMany, 1024);
115 }
116
117 static Cell[] getCells(final int howMany, final int valueSize) {
118 Cell[] cells = new Cell[howMany];
119 byte[] value = new byte[valueSize];
120 for (int i = 0; i < howMany; i++) {
121 byte[] index = Bytes.toBytes(i);
122 KeyValue kv = new KeyValue(index, Bytes.toBytes("f"), index, value);
123 cells[i] = kv;
124 }
125 return cells;
126 }
127
128 private static final String COUNT = "--count=";
129 private static final String SIZE = "--size=";
130
131
132
133
134
135 private static void usage(final int errorCode) {
136 System.out.println("Usage: IPCUtil [options]");
137 System.out.println("Micro-benchmarking how changed sizes and counts work with buffer resizing");
138 System.out.println(" --count Count of Cells");
139 System.out.println(" --size Size of Cell values");
140 System.out.println("Example: IPCUtil --count=1024 --size=1024");
141 System.exit(errorCode);
142 }
143
144 private static void timerTests(final CellBlockBuilder builder, final int count, final int size,
145 final Codec codec, final CompressionCodec compressor) throws IOException {
146 final int cycles = 1000;
147 StopWatch timer = new StopWatch();
148 timer.start();
149 for (int i = 0; i < cycles; i++) {
150 timerTest(builder, timer, count, size, codec, compressor, false);
151 }
152 timer.stop();
153 LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + false + ", count=" +
154 count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
155 timer.reset();
156 timer.start();
157 for (int i = 0; i < cycles; i++) {
158 timerTest(builder, timer, count, size, codec, compressor, true);
159 }
160 timer.stop();
161 LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + true + ", count=" +
162 count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
163 }
164
165 private static void timerTest(final CellBlockBuilder builder, final StopWatch timer,
166 final int count, final int size, final Codec codec, final CompressionCodec compressor,
167 final boolean sized) throws IOException {
168 doBuildCellBlockUndoCellBlock(builder, codec, compressor, count, size, sized);
169 }
170
171
172
173
174
175
176
177 public static void main(String[] args) throws IOException {
178 int count = 1024;
179 int size = 10240;
180 for (String arg : args) {
181 if (arg.startsWith(COUNT)) {
182 count = Integer.parseInt(arg.replace(COUNT, ""));
183 } else if (arg.startsWith(SIZE)) {
184 size = Integer.parseInt(arg.replace(SIZE, ""));
185 } else {
186 usage(1);
187 }
188 }
189 CellBlockBuilder builder = new CellBlockBuilder(HBaseConfiguration.create());
190 ((Log4JLogger) CellBlockBuilder.LOG).getLogger().setLevel(Level.ALL);
191 timerTests(builder, count, size, new KeyValueCodec(), null);
192 timerTests(builder, count, size, new KeyValueCodec(), new DefaultCodec());
193 timerTests(builder, count, size, new KeyValueCodec(), new GzipCodec());
194 }
195 }