1
2
3
4
5
6
7
8
9
10
11 package org.apache.hadoop.hbase.namespace;
12
13 import static org.junit.Assert.assertEquals;
14 import static org.junit.Assert.assertFalse;
15 import static org.junit.Assert.assertNotNull;
16 import static org.junit.Assert.assertNull;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.IOException;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Set;
24 import java.util.concurrent.CountDownLatch;
25
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.CategoryBasedTimeout;
33 import org.apache.hadoop.hbase.Coprocessor;
34 import org.apache.hadoop.hbase.CoprocessorEnvironment;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.NamespaceDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.Waiter;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.ConnectionFactory;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.client.Mutation;
50 import org.apache.hadoop.hbase.client.RegionLocator;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54 import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56 import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59 import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60 import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.master.RegionState;
64 import org.apache.hadoop.hbase.master.RegionStates;
65 import org.apache.hadoop.hbase.master.TableNamespaceManager;
66 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67 import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68 import org.apache.hadoop.hbase.quotas.QuotaUtil;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.Region;
72 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74 import org.apache.hadoop.hbase.testclassification.MediumTests;
75 import org.apache.hadoop.hbase.util.Bytes;
76 import org.apache.hadoop.hbase.util.FSUtils;
77 import org.apache.zookeeper.KeeperException;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.BeforeClass;
81 import org.junit.Ignore;
82 import org.junit.Rule;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.junit.rules.TestRule;
86
87 import com.google.common.collect.Sets;
88
89 @Category(MediumTests.class)
90 public class TestNamespaceAuditor {
91 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93 private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95 private static HBaseAdmin ADMIN;
96 private String prefix = "TestNamespaceAuditor";
97
98 @BeforeClass
99 public static void before() throws Exception {
100 UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101 setupOnce();
102 }
103
104 public static void setupOnce() throws Exception, IOException {
105 Configuration conf = UTIL.getConfiguration();
106 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107 conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110 conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111 RegionServerObserver.class);
112 UTIL.startMiniCluster(1, 1);
113 waitForQuotaEnabled();
114 ADMIN = UTIL.getHBaseAdmin();
115 }
116
117 @AfterClass
118 public static void tearDown() throws Exception {
119 UTIL.shutdownMiniCluster();
120 }
121
122 @After
123 public void cleanup() throws Exception, KeeperException {
124 for (HTableDescriptor table : ADMIN.listTables()) {
125 ADMIN.disableTable(table.getTableName());
126 deleteTable(table.getTableName());
127 }
128 for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129 if (ns.getName().startsWith(prefix)) {
130 ADMIN.deleteNamespace(ns.getName());
131 }
132 }
133 assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
134 .getMasterQuotaManager().isQuotaInitialized());
135 }
136
137 @Test
138 public void testTableOperations() throws Exception {
139 String nsp = prefix + "_np2";
140 NamespaceDescriptor nspDesc =
141 NamespaceDescriptor.create(nsp)
142 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144 ADMIN.createNamespace(nspDesc);
145 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146 assertEquals(ADMIN.listNamespaces().length, 3);
147 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
148 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
149
150 HTableDescriptor tableDescOne =
151 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
152 tableDescOne.addFamily(fam1);
153 HTableDescriptor tableDescTwo =
154 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
155 tableDescTwo.addFamily(fam1);
156 HTableDescriptor tableDescThree =
157 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
158 tableDescThree.addFamily(fam1);
159 ADMIN.createTable(tableDescOne);
160 boolean constraintViolated = false;
161 try {
162 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
163 } catch (Exception exp) {
164 assertTrue(exp instanceof IOException);
165 constraintViolated = true;
166 } finally {
167 assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
168 constraintViolated);
169 }
170 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
171 NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
172 assertNotNull(nspState);
173 assertTrue(nspState.getTables().size() == 2);
174 assertTrue(nspState.getRegionCount() == 5);
175 constraintViolated = false;
176 try {
177 ADMIN.createTable(tableDescThree);
178 } catch (Exception exp) {
179 assertTrue(exp instanceof IOException);
180 constraintViolated = true;
181 } finally {
182 assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
183 constraintViolated);
184 }
185 }
186
187 @Test
188 public void testValidQuotas() throws Exception {
189 boolean exceptionCaught = false;
190 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
191 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
192 NamespaceDescriptor nspDesc =
193 NamespaceDescriptor.create(prefix + "vq1")
194 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
195 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
196 try {
197 ADMIN.createNamespace(nspDesc);
198 } catch (Exception exp) {
199 LOG.warn(exp);
200 exceptionCaught = true;
201 } finally {
202 assertTrue(exceptionCaught);
203 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
204 }
205 nspDesc =
206 NamespaceDescriptor.create(prefix + "vq2")
207 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
208 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
209 try {
210 ADMIN.createNamespace(nspDesc);
211 } catch (Exception exp) {
212 LOG.warn(exp);
213 exceptionCaught = true;
214 } finally {
215 assertTrue(exceptionCaught);
216 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
217 }
218 nspDesc =
219 NamespaceDescriptor.create(prefix + "vq3")
220 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
221 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
222 try {
223 ADMIN.createNamespace(nspDesc);
224 } catch (Exception exp) {
225 LOG.warn(exp);
226 exceptionCaught = true;
227 } finally {
228 assertTrue(exceptionCaught);
229 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
230 }
231 nspDesc =
232 NamespaceDescriptor.create(prefix + "vq4")
233 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
234 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
235 try {
236 ADMIN.createNamespace(nspDesc);
237 } catch (Exception exp) {
238 LOG.warn(exp);
239 exceptionCaught = true;
240 } finally {
241 assertTrue(exceptionCaught);
242 assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
243 }
244 }
245
246 @Test
247 public void testDeleteTable() throws Exception {
248 String namespace = prefix + "_dummy";
249 NamespaceDescriptor nspDesc =
250 NamespaceDescriptor.create(namespace)
251 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
252 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
253 ADMIN.createNamespace(nspDesc);
254 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
255 NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
256 assertNotNull("Namespace state found null for " + namespace, stateInfo);
257 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
258 HTableDescriptor tableDescOne =
259 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
260 tableDescOne.addFamily(fam1);
261 HTableDescriptor tableDescTwo =
262 new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
263 tableDescTwo.addFamily(fam1);
264 ADMIN.createTable(tableDescOne);
265 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
266 stateInfo = getNamespaceState(nspDesc.getName());
267 assertNotNull("Namespace state found to be null.", stateInfo);
268 assertEquals(2, stateInfo.getTables().size());
269 assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
270 assertEquals(6, stateInfo.getRegionCount());
271 ADMIN.disableTable(tableDescOne.getTableName());
272 deleteTable(tableDescOne.getTableName());
273 stateInfo = getNamespaceState(nspDesc.getName());
274 assertNotNull("Namespace state found to be null.", stateInfo);
275 assertEquals(5, stateInfo.getRegionCount());
276 assertEquals(1, stateInfo.getTables().size());
277 ADMIN.disableTable(tableDescTwo.getTableName());
278 deleteTable(tableDescTwo.getTableName());
279 ADMIN.deleteNamespace(namespace);
280 stateInfo = getNamespaceState(namespace);
281 assertNull("Namespace state not found to be null.", stateInfo);
282 }
283
284 public static class CPRegionServerObserver extends BaseRegionServerObserver {
285 private volatile boolean shouldFailMerge = false;
286
287 public void failMerge(boolean fail) {
288 shouldFailMerge = fail;
289 }
290
291 private boolean triggered = false;
292
293 public synchronized void waitUtilTriggered() throws InterruptedException {
294 while (!triggered) {
295 wait();
296 }
297 }
298
299 @Override
300 public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
301 Region regionA, Region regionB) throws IOException {
302 triggered = true;
303 notifyAll();
304 if (shouldFailMerge) {
305 throw new IOException("fail merge");
306 }
307 }
308 }
309
310 @Test
311 public void testRegionMerge() throws Exception {
312 String nsp1 = prefix + "_regiontest";
313 NamespaceDescriptor nspDesc =
314 NamespaceDescriptor.create(nsp1)
315 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
316 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
317 ADMIN.createNamespace(nspDesc);
318 final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
319 byte[] columnFamily = Bytes.toBytes("info");
320 HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
321 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
322 final int initialRegions = 3;
323 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
324 try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
325 Table table = connection.getTable(tableTwo)) {
326 UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
327 }
328 ADMIN.flush(tableTwo);
329 List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
330 Collections.sort(hris);
331
332 final Set<String> encodedRegionNamesToMerge =
333 Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
334 ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
335 false);
336 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
337 hris = ADMIN.getTableRegions(tableTwo);
338 assertEquals(initialRegions - 1, hris.size());
339 Collections.sort(hris);
340
341 final HRegionInfo hriToSplit = hris.get(1);
342 ADMIN.split(tableTwo, Bytes.toBytes("500"));
343
344 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
345
346 @Override
347 public boolean evaluate() throws Exception {
348 RegionStates regionStates =
349 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
350 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
351 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
352 return false;
353 }
354 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
355 return false;
356 }
357 }
358 return true;
359 }
360
361 @Override
362 public String explainFailure() throws Exception {
363 RegionStates regionStates =
364 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
365 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
366 if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
367 return hriToSplit + " which is expected to be split is still online";
368 }
369 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
370 return hri + " is still in not opened";
371 }
372 }
373 return "Unknown";
374 }
375 });
376 hris = ADMIN.getTableRegions(tableTwo);
377 assertEquals(initialRegions, hris.size());
378 Collections.sort(hris);
379
380
381 MiniHBaseCluster cluster = UTIL.getHBaseCluster();
382 HRegionServer regionServer = cluster.getRegionServer(0);
383 RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
384 Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
385 CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
386 regionServerObserver.failMerge(true);
387 regionServerObserver.triggered = false;
388
389 ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
390 false);
391 regionServerObserver.waitUtilTriggered();
392 hris = ADMIN.getTableRegions(tableTwo);
393 assertEquals(initialRegions, hris.size());
394 Collections.sort(hris);
395
396 ADMIN.split(tableTwo, Bytes.toBytes("6"));
397 waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
398 assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
399 }
400
401 private void waitForMergeToComplete(final TableName tableTwo,
402 final Set<String> encodedRegionNamesToMerge) throws Exception {
403 UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
404
405 @Override
406 public boolean evaluate() throws Exception {
407 RegionStates regionStates =
408 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
409 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
410 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
411 return false;
412 }
413 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
414 return false;
415 }
416 }
417 return true;
418 }
419
420 @Override
421 public String explainFailure() throws Exception {
422 RegionStates regionStates =
423 UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
424 for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
425 if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
426 return hri + " which is expected to be merged is still online";
427 }
428 if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
429 return hri + " is still in not opened";
430 }
431 }
432 return "Unknown";
433 }
434 });
435 }
436
437 @Ignore("Hangs on occasion waiting on countdown latch") @Test
438 public void testRegionOperations() throws Exception {
439 String nsp1 = prefix + "_regiontest";
440 NamespaceDescriptor nspDesc =
441 NamespaceDescriptor.create(nsp1)
442 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
443 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
444 ADMIN.createNamespace(nspDesc);
445 boolean constraintViolated = false;
446 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
447 byte[] columnFamily = Bytes.toBytes("info");
448 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
449 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
450 NamespaceTableAndRegionInfo stateInfo;
451 try {
452 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
453 } catch (Exception exp) {
454 assertTrue(exp instanceof DoNotRetryIOException);
455 LOG.info(exp);
456 constraintViolated = true;
457 } finally {
458 assertTrue(constraintViolated);
459 }
460 assertFalse(ADMIN.tableExists(tableOne));
461
462 ADMIN.createTable(tableDescOne);
463 Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
464 HTable htable = (HTable) connection.getTable(tableOne);
465 UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
466 ADMIN.flush(tableOne);
467 stateInfo = getNamespaceState(nsp1);
468 assertEquals(1, stateInfo.getTables().size());
469 assertEquals(1, stateInfo.getRegionCount());
470 restartMaster();
471 ADMIN.split(tableOne, Bytes.toBytes("500"));
472 HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473 CustomObserver observer =
474 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475 CustomObserver.class.getName());
476 assertNotNull(observer);
477 observer.postSplit.await();
478 assertEquals(2, ADMIN.getTableRegions(tableOne).size());
479 actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
480 observer =
481 (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
482 CustomObserver.class.getName());
483 assertNotNull(observer);
484 ADMIN.split(
485 tableOne,
486 getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
487 .getEndKey()));
488 observer.postSplit.await();
489
490 List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
491 assertEquals(2, hris.size());
492 assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
493
494 htable.close();
495 }
496
497
498
499
500
501
502 @Test
503 public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
504 String nsp1 = prefix + "_testRecreateTable";
505 NamespaceDescriptor nspDesc =
506 NamespaceDescriptor.create(nsp1)
507 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
508 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
509 ADMIN.createNamespace(nspDesc);
510 final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
511 byte[] columnFamily = Bytes.toBytes("info");
512 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
513 tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
514 MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
515 try {
516 try {
517 ADMIN.createTable(tableDescOne);
518 fail("Table " + tableOne.toString() + "creation should fail.");
519 } catch (Exception exp) {
520 LOG.error(exp);
521 }
522 assertFalse(ADMIN.tableExists(tableOne));
523
524 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
525 assertEquals("First table creation failed in namespace so number of tables in namespace "
526 + "should be 0.", 0, nstate.getTables().size());
527
528 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
529 try {
530 ADMIN.createTable(tableDescOne);
531 } catch (Exception e) {
532 fail("Table " + tableOne.toString() + "creation should succeed.");
533 LOG.error(e);
534 }
535 assertTrue(ADMIN.tableExists(tableOne));
536 nstate = getNamespaceState(nsp1);
537 assertEquals("First table was created successfully so table size in namespace should "
538 + "be one now.", 1, nstate.getTables().size());
539 } finally {
540 MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
541 if (ADMIN.tableExists(tableOne)) {
542 ADMIN.disableTable(tableOne);
543 deleteTable(tableOne);
544 }
545 ADMIN.deleteNamespace(nsp1);
546 }
547 }
548
549 private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
550 IOException {
551 return getQuotaManager().getState(namespace);
552 }
553
554 byte[] getSplitKey(byte[] startKey, byte[] endKey) {
555 String skey = Bytes.toString(startKey);
556 int key;
557 if (StringUtils.isBlank(skey)) {
558 key = Integer.parseInt(Bytes.toString(endKey)) / 2;
559 } else {
560 key = (int) (Integer.parseInt(skey) * 1.5);
561 }
562 return Bytes.toBytes("" + key);
563 }
564
565 public static class CustomObserver extends BaseRegionObserver {
566 volatile CountDownLatch postSplit;
567 volatile CountDownLatch preSplitBeforePONR;
568
569 @Override
570 public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
571 throws IOException {
572 postSplit.countDown();
573 }
574
575 @Override
576 public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
577 byte[] splitKey, List<Mutation> metaEntries) throws IOException {
578 preSplitBeforePONR.countDown();
579 }
580
581 @Override
582 public void start(CoprocessorEnvironment e) throws IOException {
583 postSplit = new CountDownLatch(1);
584 preSplitBeforePONR = new CountDownLatch(1);
585 }
586 }
587
588 @Test
589 public void testStatePreserve() throws Exception {
590 final String nsp1 = prefix + "_testStatePreserve";
591 NamespaceDescriptor nspDesc =
592 NamespaceDescriptor.create(nsp1)
593 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
594 .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
595 ADMIN.createNamespace(nspDesc);
596 TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
597 TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
598 TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
599 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
600 HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
601 tableDescOne.addFamily(fam1);
602 HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
603 tableDescTwo.addFamily(fam1);
604 HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
605 tableDescThree.addFamily(fam1);
606 ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
607 ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
608 ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
609 ADMIN.disableTable(tableThree);
610 deleteTable(tableThree);
611
612 UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
613 @Override
614 public boolean evaluate() throws Exception {
615 return (getNamespaceState(nsp1).getTables().size() == 2);
616 }
617 });
618 NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
619 restartMaster();
620 NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
621 assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
622 .getTables().size(), after.getTables().size());
623 }
624
625 private static void waitForQuotaEnabled() throws Exception {
626 UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
627 @Override
628 public boolean evaluate() throws Exception {
629 HMaster master = UTIL.getHBaseCluster().getMaster();
630 if (master == null) {
631 return false;
632 }
633 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
634 return quotaManager != null && quotaManager.isQuotaInitialized();
635 }
636 });
637 }
638
639 private void restartMaster() throws Exception {
640 UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
641 UTIL.getHBaseCluster().waitOnMaster(0);
642 UTIL.getHBaseCluster().startMaster();
643 waitForQuotaEnabled();
644 }
645
646 private NamespaceAuditor getQuotaManager() {
647 return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
648 }
649
650 public static class MasterSyncObserver extends BaseMasterObserver {
651 volatile CountDownLatch tableDeletionLatch;
652 static boolean throwExceptionInPreCreateTableHandler;
653
654 @Override
655 public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
656 TableName tableName) throws IOException {
657 tableDeletionLatch = new CountDownLatch(1);
658 }
659
660 @Override
661 public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
662 TableName tableName) throws IOException {
663 tableDeletionLatch.countDown();
664 }
665
666 @Override
667 public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
668 HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
669 if (throwExceptionInPreCreateTableHandler) {
670 throw new IOException("Throw exception as it is demanded.");
671 }
672 }
673 }
674
675 private void deleteTable(final TableName tableName) throws Exception {
676
677
678 MasterSyncObserver observer =
679 (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
680 .findCoprocessor(MasterSyncObserver.class.getName());
681 ADMIN.deleteTable(tableName);
682 observer.tableDeletionLatch.await();
683 }
684
685 @Test(expected = QuotaExceededException.class)
686 public void testExceedTableQuotaInNamespace() throws Exception {
687 String nsp = prefix + "_testExceedTableQuotaInNamespace";
688 NamespaceDescriptor nspDesc =
689 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
690 .build();
691 ADMIN.createNamespace(nspDesc);
692 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
693 assertEquals(ADMIN.listNamespaces().length, 3);
694 assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
695 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
696 HTableDescriptor tableDescOne =
697 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
698 tableDescOne.addFamily(fam1);
699 HTableDescriptor tableDescTwo =
700 new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701 tableDescTwo.addFamily(fam1);
702 ADMIN.createTable(tableDescOne);
703 ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
704 }
705
706 @Test(expected = QuotaExceededException.class)
707 public void testCloneSnapshotQuotaExceed() throws Exception {
708 String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
709 NamespaceDescriptor nspDesc =
710 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
711 .build();
712 ADMIN.createNamespace(nspDesc);
713 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
714 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
715 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
716 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
717 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
718 tableDescOne.addFamily(fam1);
719 ADMIN.createTable(tableDescOne);
720 String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
721 ADMIN.snapshot(snapshot, tableName);
722 ADMIN.cloneSnapshot(snapshot, cloneTableName);
723 ADMIN.deleteSnapshot(snapshot);
724 }
725
726 @Test
727 public void testCloneSnapshot() throws Exception {
728 String nsp = prefix + "_testCloneSnapshot";
729 NamespaceDescriptor nspDesc =
730 NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
731 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
732 ADMIN.createNamespace(nspDesc);
733 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
734 TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
735 TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
736
737 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
738 HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
739 tableDescOne.addFamily(fam1);
740
741 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
742 String snapshot = "snapshot_testCloneSnapshot";
743 ADMIN.snapshot(snapshot, tableName);
744 ADMIN.cloneSnapshot(snapshot, cloneTableName);
745
746 int tableLength;
747 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
748 tableLength = locator.getStartKeys().length;
749 }
750 assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
751
752 try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
753 tableLength = locator.getStartKeys().length;
754 }
755 assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
756
757 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
758 assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
759 assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
760
761 ADMIN.deleteSnapshot(snapshot);
762 }
763
764 @Test
765 public void testRestoreSnapshot() throws Exception {
766 String nsp = prefix + "_testRestoreSnapshot";
767 NamespaceDescriptor nspDesc =
768 NamespaceDescriptor.create(nsp)
769 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
770 ADMIN.createNamespace(nspDesc);
771 assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
772 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
773 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
774 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
775 tableDescOne.addFamily(fam1);
776 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
777
778 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
779 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
780
781 String snapshot = "snapshot_testRestoreSnapshot";
782 ADMIN.snapshot(snapshot, tableName1);
783
784 List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
785 Collections.sort(regions);
786
787 ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
788 Thread.sleep(2000);
789 assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
790
791 ADMIN.disableTable(tableName1);
792 ADMIN.restoreSnapshot(snapshot);
793
794 assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
795
796 ADMIN.enableTable(tableName1);
797 ADMIN.deleteSnapshot(snapshot);
798 }
799
800 @Test
801 public void testRestoreSnapshotQuotaExceed() throws Exception {
802 String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
803 NamespaceDescriptor nspDesc =
804 NamespaceDescriptor.create(nsp)
805 .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
806 ADMIN.createNamespace(nspDesc);
807 NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
808 assertNotNull("Namespace descriptor found null.", ndesc);
809 TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
810 HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
811 HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
812 tableDescOne.addFamily(fam1);
813
814 ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
815
816 NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
817 assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
818
819 String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
820
821 ADMIN.snapshot(snapshot, tableName1);
822
823 ADMIN.disableTable(tableName1);
824 ADMIN.deleteTable(tableName1);
825 ADMIN.createTable(tableDescOne);
826 ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
827 ADMIN.modifyNamespace(ndesc);
828
829 ADMIN.disableTable(tableName1);
830 try {
831 ADMIN.restoreSnapshot(snapshot);
832 fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
833 + " wraps IOException into RestoreSnapshotException");
834 } catch (RestoreSnapshotException ignore) {
835 assertTrue(ignore.getCause() instanceof QuotaExceededException);
836 }
837 assertEquals(1, getNamespaceState(nsp).getRegionCount());
838 ADMIN.enableTable(tableName1);
839 ADMIN.deleteSnapshot(snapshot);
840 }
841 }