1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import com.google.protobuf.RpcController;
21 import com.google.protobuf.ServiceException;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.hbase.*;
24
25 import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
26 import org.apache.hadoop.hbase.exceptions.LockTimeoutException;
27 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
28 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
29 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
30 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
31 import org.apache.hadoop.hbase.quotas.ThrottlingException;
32 import org.apache.hadoop.hbase.regionserver.HRegionServer;
33 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
34 import org.apache.hadoop.hbase.regionserver.Region;
35 import org.apache.hadoop.hbase.testclassification.ClientTests;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 import java.io.IOException;
44 import java.util.ArrayList;
45 import java.util.List;
46 import org.slf4j.Logger;
47 import org.slf4j.LoggerFactory;
48
49 import static junit.framework.Assert.assertEquals;
50 import static org.junit.Assert.assertNotNull;
51 import static org.junit.Assert.assertNull;
52 import static org.junit.Assert.assertTrue;
53 import static org.junit.Assert.fail;
54
55 @Category({MediumTests.class, ClientTests.class})
56 public class TestMetaCache {
57 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58 private static final TableName TABLE_NAME = TableName.valueOf("test_table");
59 private static final byte[] FAMILY = Bytes.toBytes("fam1");
60 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
61
62 private static HRegionServer badRS;
63 private static final Logger LOG = LoggerFactory.getLogger(TestMetaCache.class);
64
65
66
67
68 @BeforeClass
69 public static void setUpBeforeClass() throws Exception {
70 Configuration conf = TEST_UTIL.getConfiguration();
71 conf.setStrings(HConstants.REGION_SERVER_IMPL,
72 RegionServerWithFakeRpcServices.class.getName());
73 TEST_UTIL.startMiniCluster(1);
74 TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster();
75 TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME.META_TABLE_NAME);
76 badRS = TEST_UTIL.getHBaseCluster().getRegionServer(0);
77 assertTrue(badRS.getRSRpcServices() instanceof FakeRSRpcServices);
78 HTableDescriptor table = new HTableDescriptor(TABLE_NAME);
79 HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
80 fam.setMaxVersions(2);
81 table.addFamily(fam);
82 TEST_UTIL.createTable(table, null);
83 }
84
85
86
87
88
89 @AfterClass
90 public static void tearDownAfterClass() throws Exception {
91 TEST_UTIL.shutdownMiniCluster();
92 }
93
94 @Test
95 public void testPreserveMetaCacheOnException() throws Exception {
96 ((FakeRSRpcServices)badRS.getRSRpcServices()).setExceptionInjector(
97 new RoundRobinExceptionInjector());
98 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
99 conf.set("hbase.client.retries.number", "1");
100 ConnectionManager.HConnectionImplementation conn =
101 (ConnectionManager.HConnectionImplementation) ConnectionFactory.createConnection(conf);
102 try {
103 Table table = conn.getTable(TABLE_NAME);
104 byte[] row = Bytes.toBytes("row1");
105
106 Put put = new Put(row);
107 put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10));
108 Get get = new Get(row);
109 Append append = new Append(row);
110 append.add(FAMILY, QUALIFIER, Bytes.toBytes(11));
111 Increment increment = new Increment(row);
112 increment.addColumn(FAMILY, QUALIFIER, 10);
113 Delete delete = new Delete(row);
114 delete.addColumn(FAMILY, QUALIFIER);
115 RowMutations mutations = new RowMutations(row);
116 mutations.add(put);
117 mutations.add(delete);
118
119 Exception exp;
120 boolean success;
121 for (int i = 0; i < 50; i++) {
122 exp = null;
123 success = false;
124 try {
125 table.put(put);
126
127 success = true;
128 table.get(get);
129 table.append(append);
130 table.increment(increment);
131 table.delete(delete);
132 table.mutateRow(mutations);
133 } catch (IOException ex) {
134
135 if (ClientExceptionsUtil.isMetaClearingException(ex) || success) {
136 exp = ex;
137 }
138 }
139
140 if (exp != null && ClientExceptionsUtil.isMetaClearingException(exp)) {
141 assertNull(conn.getCachedLocation(TABLE_NAME, row));
142 } else if (success) {
143 assertNotNull(conn.getCachedLocation(TABLE_NAME, row));
144 }
145 }
146 } finally {
147 conn.close();
148 }
149 }
150
151 @Test
152 public void testCacheClearingOnCallQueueTooBig() throws Exception {
153 ((FakeRSRpcServices)badRS.getRSRpcServices()).setExceptionInjector(
154 new CallQueueTooBigExceptionInjector());
155 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
156 conf.set("hbase.client.retries.number", "2");
157 conf.set(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, "true");
158 ConnectionManager.HConnectionImplementation conn =
159 (ConnectionManager.HConnectionImplementation) ConnectionFactory.createConnection(conf);
160 try {
161 Table table = conn.getTable(TABLE_NAME);
162 byte[] row = Bytes.toBytes("row1");
163
164 Put put = new Put(row);
165 put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10));
166 table.put(put);
167
168
169 MetricsConnection metrics = conn.getConnectionMetrics();
170 long preGetRegionClears = metrics.metaCacheNumClearRegion.count();
171 long preGetServerClears = metrics.metaCacheNumClearServer.count();
172
173
174 Get get = new Get(row);
175 try {
176 table.get(get);
177 fail("Expected CallQueueTooBigException");
178 } catch (RetriesExhaustedException ree) {
179
180 }
181
182
183 long postGetRegionClears = metrics.metaCacheNumClearRegion.count();
184 long postGetServerClears = metrics.metaCacheNumClearServer.count();
185 assertEquals(preGetRegionClears, postGetRegionClears);
186 assertEquals(preGetServerClears, postGetServerClears);
187 } finally {
188 conn.close();
189 }
190 }
191
192 public static List<Throwable> metaCachePreservingExceptions() {
193 return new ArrayList<Throwable>() {{
194 add(new RegionOpeningException(" "));
195 add(new RegionTooBusyException());
196 add(new ThrottlingException(" "));
197 add(new MultiActionResultTooLarge(" "));
198 add(new RetryImmediatelyException(" "));
199 add(new CallQueueTooBigException());
200 }};
201 }
202
203 public static class RegionServerWithFakeRpcServices extends HRegionServer {
204 private FakeRSRpcServices rsRpcServices;
205
206 public RegionServerWithFakeRpcServices(Configuration conf, CoordinatedStateManager cp)
207 throws IOException, InterruptedException {
208 super(conf, cp);
209 }
210
211 @Override
212 protected RSRpcServices createRpcServices() throws IOException {
213 this.rsRpcServices = new FakeRSRpcServices(this);
214 return rsRpcServices;
215 }
216
217 public void setExceptionInjector(ExceptionInjector injector) {
218 rsRpcServices.setExceptionInjector(injector);
219 }
220 }
221
222 public static class FakeRSRpcServices extends RSRpcServices {
223
224 private ExceptionInjector exceptions;
225
226 public FakeRSRpcServices(HRegionServer rs) throws IOException {
227 super(rs);
228 exceptions = new RoundRobinExceptionInjector();
229 }
230
231 public void setExceptionInjector(ExceptionInjector injector) {
232 this.exceptions = injector;
233 }
234
235 @Override
236 public GetResponse get(final RpcController controller,
237 final ClientProtos.GetRequest request) throws ServiceException {
238 exceptions.throwOnGet(this, request);
239 return super.get(controller, request);
240 }
241
242 @Override
243 public ClientProtos.MutateResponse mutate(final RpcController controller,
244 final ClientProtos.MutateRequest request) throws ServiceException {
245 exceptions.throwOnMutate(this, request);
246 return super.mutate(controller, request);
247 }
248
249 @Override
250 public ClientProtos.ScanResponse scan(final RpcController controller,
251 final ClientProtos.ScanRequest request) throws ServiceException {
252 exceptions.throwOnScan(this, request);
253 return super.scan(controller, request);
254 }
255
256 public Region getRegion(
257 final HBaseProtos.RegionSpecifier regionSpecifier) throws IOException {
258 return super.getRegion(regionSpecifier);
259 }
260 }
261
262 public static abstract class ExceptionInjector {
263 protected boolean isTestTable(FakeRSRpcServices rpcServices,
264 HBaseProtos.RegionSpecifier regionSpec) throws ServiceException {
265 try {
266 return TABLE_NAME.equals(
267 rpcServices.getRegion(regionSpec).getTableDesc().getTableName());
268 } catch (IOException ioe) {
269 throw new ServiceException(ioe);
270 }
271 }
272
273 public abstract void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
274 throws ServiceException;
275
276 public abstract void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
277 throws ServiceException;
278
279 public abstract void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
280 throws ServiceException;
281 }
282
283
284
285
286
287 public static class RoundRobinExceptionInjector extends ExceptionInjector {
288 private int numReqs = -1;
289 private int expCount = -1;
290 private List<Throwable> metaCachePreservingExceptions = metaCachePreservingExceptions();
291
292 public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
293 throws ServiceException {
294 throwSomeExceptions(rpcServices, request.getRegion());
295 }
296
297 public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
298 throws ServiceException {
299 throwSomeExceptions(rpcServices, request.getRegion());
300 }
301
302 public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
303 throws ServiceException {
304 if (!request.hasScannerId()) {
305
306 throwSomeExceptions(rpcServices, request.getRegion());
307 }
308 }
309
310
311
312
313
314
315 private void throwSomeExceptions(FakeRSRpcServices rpcServices,
316 HBaseProtos.RegionSpecifier regionSpec)
317 throws ServiceException {
318 if (!isTestTable(rpcServices, regionSpec)) {
319 return;
320 }
321
322 numReqs++;
323
324
325 if (numReqs % 5 ==0) {
326 return;
327 } else if (numReqs % 5 == 1 || numReqs % 5 == 2) {
328 throw new ServiceException(new NotServingRegionException());
329 }
330
331
332
333
334 expCount++;
335 Throwable t = metaCachePreservingExceptions.get(
336 expCount % metaCachePreservingExceptions.size());
337 throw new ServiceException(t);
338 }
339 }
340
341
342
343
344 public static class CallQueueTooBigExceptionInjector extends ExceptionInjector {
345 @Override
346 public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
347 throws ServiceException {
348 if (isTestTable(rpcServices, request.getRegion())) {
349 throw new ServiceException(new CallQueueTooBigException());
350 }
351 }
352
353 @Override
354 public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
355 throws ServiceException {
356 }
357
358 @Override
359 public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
360 throws ServiceException {
361 }
362 }
363
364 @Test
365 public void testUserRegionLockThrowsException() throws IOException, InterruptedException {
366 ((FakeRSRpcServices)badRS.getRSRpcServices()).setExceptionInjector(new LockSleepInjector());
367 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
368 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
369 conf.setLong(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 2000);
370 conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 2000);
371
372 try (ConnectionManager.HConnectionImplementation conn =
373 (ConnectionManager.HConnectionImplementation) ConnectionFactory.createConnection(conf)) {
374 ClientThread client1 = new ClientThread(conn);
375 ClientThread client2 = new ClientThread(conn);
376 client1.start();
377 client2.start();
378 client1.join();
379 client2.join();
380
381
382
383
384
385
386
387
388
389
390 assertNotNull(client1.getException());
391 assertNotNull(client2.getException());
392
393 assertTrue(client1.getException() instanceof LockTimeoutException
394 ^ client2.getException() instanceof LockTimeoutException);
395 }
396 }
397
398 private final class ClientThread extends Thread {
399 private Exception exception;
400 private ConnectionManager.HConnectionImplementation connection;
401
402 private ClientThread(ConnectionManager.HConnectionImplementation connection) {
403 this.connection = connection;
404 }
405 @Override
406 public void run() {
407 byte[] currentKey = HConstants.EMPTY_START_ROW;
408 try {
409 connection.getRegionLocation(TABLE_NAME, currentKey, true);
410 } catch (IOException e) {
411 LOG.error("Thread id: " + this.getId() + " exception: ", e);
412 this.exception = e;
413 }
414 }
415 public Exception getException() {
416 return exception;
417 }
418 }
419
420 public static class LockSleepInjector extends ExceptionInjector {
421 @Override
422 public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request) {
423 try {
424 Thread.sleep(5000);
425 } catch (InterruptedException e) {
426 LOG.info("Interrupted exception", e);
427 }
428 }
429
430 @Override
431 public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request) { }
432
433 @Override
434 public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request) { }
435 }
436 }