1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_END_KEY;
21 import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_START_KEY;
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertNotNull;
26 import static org.junit.Assert.assertNull;
27 import static org.junit.Assert.assertTrue;
28 import static org.mockito.Matchers.any;
29 import static org.mockito.Matchers.anyBoolean;
30 import static org.mockito.Matchers.anyCollection;
31 import static org.mockito.Matchers.anyLong;
32 import static org.mockito.Mockito.doAnswer;
33 import static org.mockito.Mockito.mock;
34 import static org.mockito.Mockito.when;
35
36 import java.io.IOException;
37 import java.util.ArrayList;
38 import java.util.Arrays;
39 import java.util.List;
40 import java.util.TreeMap;
41
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.hbase.Cell;
44 import org.apache.hadoop.hbase.KeyValue;
45 import org.apache.hadoop.hbase.io.hfile.HFile;
46 import org.apache.hadoop.hbase.regionserver.BloomType;
47 import org.apache.hadoop.hbase.regionserver.InternalScanner;
48 import org.apache.hadoop.hbase.regionserver.ScannerContext;
49 import org.apache.hadoop.hbase.regionserver.StoreFile;
50 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
51 import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.mockito.invocation.InvocationOnMock;
54 import org.mockito.stubbing.Answer;
55
56 public class TestCompactor {
57
58 public static StoreFile createDummyStoreFile(long maxSequenceId) throws Exception {
59
60
61 StoreFile sf = mock(StoreFile.class);
62 StoreFile.Reader r = mock(StoreFile.Reader.class);
63 when(r.length()).thenReturn(1L);
64 when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
65 when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
66 when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(),
67 anyBoolean())).thenReturn(mock(StoreFileScanner.class));
68 when(sf.getReader()).thenReturn(r);
69 when(sf.createReader()).thenReturn(r);
70 when(sf.createReader(anyBoolean())).thenReturn(r);
71 when(sf.cloneForReader()).thenReturn(sf);
72 when(sf.getMaxSequenceId()).thenReturn(maxSequenceId);
73 return sf;
74 }
75
76 public static CompactionRequest createDummyRequest() throws Exception {
77 return new CompactionRequest(Arrays.asList(createDummyStoreFile(1L)));
78 }
79
80
81 public static class StoreFileWritersCapture
82 implements Answer<StoreFile.Writer>, StripeMultiFileWriter.WriterFactory {
83 public static class Writer {
84 public ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
85 public TreeMap<byte[], byte[]> data = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
86 public boolean hasMetadata;
87 }
88
89 private List<Writer> writers = new ArrayList<Writer>();
90
91 @Override
92 public StoreFile.Writer createWriter() throws IOException {
93 final Writer realWriter = new Writer();
94 writers.add(realWriter);
95 StoreFile.Writer writer = mock(StoreFile.Writer.class);
96 doAnswer(new Answer<Object>() {
97 public Object answer(InvocationOnMock invocation) {
98 return realWriter.kvs.add((KeyValue) invocation.getArguments()[0]);
99 }
100 }).when(writer).append(any(KeyValue.class));
101 doAnswer(new Answer<Object>() {
102 public Object answer(InvocationOnMock invocation) {
103 Object[] args = invocation.getArguments();
104 return realWriter.data.put((byte[]) args[0], (byte[]) args[1]);
105 }
106 }).when(writer).appendFileInfo(any(byte[].class), any(byte[].class));
107 doAnswer(new Answer<Void>() {
108 @Override
109 public Void answer(InvocationOnMock invocation) throws Throwable {
110 realWriter.hasMetadata = true;
111 return null;
112 }
113 }).when(writer).appendMetadata(any(long.class), any(boolean.class));
114 doAnswer(new Answer<Void>() {
115 @Override
116 public Void answer(InvocationOnMock invocation) throws Throwable {
117 realWriter.hasMetadata = true;
118 return null;
119 }
120 }).when(writer).appendMetadata(any(long.class), any(boolean.class), anyCollection());
121 doAnswer(new Answer<Path>() {
122 @Override
123 public Path answer(InvocationOnMock invocation) throws Throwable {
124 return new Path("foo");
125 }
126 }).when(writer).getPath();
127 return writer;
128 }
129
130 @Override
131 public StoreFile.Writer answer(InvocationOnMock invocation) throws Throwable {
132 return createWriter();
133 }
134
135 public void verifyKvs(KeyValue[][] kvss, boolean allFiles, boolean requireMetadata) {
136 if (allFiles) {
137 assertEquals(kvss.length, writers.size());
138 }
139 int skippedWriters = 0;
140 for (int i = 0; i < kvss.length; ++i) {
141 KeyValue[] kvs = kvss[i];
142 if (kvs != null) {
143 Writer w = writers.get(i - skippedWriters);
144 if (requireMetadata) {
145 assertNotNull(w.data.get(STRIPE_START_KEY));
146 assertNotNull(w.data.get(STRIPE_END_KEY));
147 } else {
148 assertNull(w.data.get(STRIPE_START_KEY));
149 assertNull(w.data.get(STRIPE_END_KEY));
150 }
151 assertEquals(kvs.length, w.kvs.size());
152 for (int j = 0; j < kvs.length; ++j) {
153 assertEquals(kvs[j], w.kvs.get(j));
154 }
155 } else {
156 assertFalse(allFiles);
157 ++skippedWriters;
158 }
159 }
160 }
161
162 public void verifyBoundaries(byte[][] boundaries) {
163 assertEquals(boundaries.length - 1, writers.size());
164 for (int i = 0; i < writers.size(); ++i) {
165 assertArrayEquals("i = " + i, boundaries[i], writers.get(i).data.get(STRIPE_START_KEY));
166 assertArrayEquals("i = " + i, boundaries[i + 1], writers.get(i).data.get(STRIPE_END_KEY));
167 }
168 }
169
170 public void verifyKvs(KeyValue[][] kvss, boolean allFiles, List<Long> boundaries) {
171 if (allFiles) {
172 assertEquals(kvss.length, writers.size());
173 }
174 int skippedWriters = 0;
175 for (int i = 0; i < kvss.length; ++i) {
176 KeyValue[] kvs = kvss[i];
177 if (kvs != null) {
178 Writer w = writers.get(i - skippedWriters);
179 assertEquals(kvs.length, w.kvs.size());
180 for (int j = 0; j < kvs.length; ++j) {
181 assertTrue(kvs[j].getTimestamp() >= boundaries.get(i));
182 assertTrue(kvs[j].getTimestamp() < boundaries.get(i + 1));
183 assertEquals(kvs[j], w.kvs.get(j));
184 }
185 } else {
186 assertFalse(allFiles);
187 ++skippedWriters;
188 }
189 }
190 }
191
192 public List<Writer> getWriters() {
193 return writers;
194 }
195 }
196
197 public static class Scanner implements InternalScanner {
198 private final ArrayList<KeyValue> kvs;
199
200 public Scanner(KeyValue... kvs) {
201 this.kvs = new ArrayList<KeyValue>(Arrays.asList(kvs));
202 }
203
204 @Override
205 public boolean next(List<Cell> results) throws IOException {
206 if (kvs.isEmpty()) return false;
207 results.add(kvs.remove(0));
208 return !kvs.isEmpty();
209 }
210
211 @Override
212 public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
213 return next(result);
214 }
215
216 @Override
217 public void close() throws IOException {
218 }
219 }
220 }