View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  package org.apache.hadoop.hbase.namespace;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertFalse;
15  import static org.junit.Assert.assertNotNull;
16  import static org.junit.Assert.assertNull;
17  import static org.junit.Assert.assertTrue;
18  import static org.junit.Assert.fail;
19  
20  import java.io.IOException;
21  import java.util.Collections;
22  import java.util.List;
23  import java.util.Set;
24  import java.util.concurrent.CountDownLatch;
25  
26  import org.apache.commons.lang.StringUtils;
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.CategoryBasedTimeout;
33  import org.apache.hadoop.hbase.Coprocessor;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MiniHBaseCluster;
42  import org.apache.hadoop.hbase.NamespaceDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.Waiter;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.ConnectionFactory;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Mutation;
50  import org.apache.hadoop.hbase.client.RegionLocator;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54  import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60  import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61  import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62  import org.apache.hadoop.hbase.master.HMaster;
63  import org.apache.hadoop.hbase.master.RegionState;
64  import org.apache.hadoop.hbase.master.RegionStates;
65  import org.apache.hadoop.hbase.master.TableNamespaceManager;
66  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67  import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68  import org.apache.hadoop.hbase.quotas.QuotaUtil;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.Region;
72  import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74  import org.apache.hadoop.hbase.testclassification.MediumTests;
75  import org.apache.hadoop.hbase.util.Bytes;
76  import org.apache.hadoop.hbase.util.FSUtils;
77  import org.apache.zookeeper.KeeperException;
78  import org.junit.After;
79  import org.junit.AfterClass;
80  import org.junit.BeforeClass;
81  import org.junit.Ignore;
82  import org.junit.Rule;
83  import org.junit.Test;
84  import org.junit.experimental.categories.Category;
85  import org.junit.rules.TestRule;
86  
87  import com.google.common.collect.Sets;
88  
89  @Category(MediumTests.class)
90  public class TestNamespaceAuditor {
91    @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92        withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93    private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95    private static HBaseAdmin ADMIN;
96    private String prefix = "TestNamespaceAuditor";
97  
98    @BeforeClass
99    public static void before() throws Exception {
100     UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101     setupOnce();
102   }
103 
104   public static void setupOnce() throws Exception, IOException {
105     Configuration conf = UTIL.getConfiguration();
106     conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109     conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111       RegionServerObserver.class);
112     UTIL.startMiniCluster(1, 1);
113     waitForQuotaEnabled();
114     ADMIN = UTIL.getHBaseAdmin();
115   }
116 
117   @AfterClass
118   public static void tearDown() throws Exception {
119     UTIL.shutdownMiniCluster();
120   }
121 
122   @After
123   public void cleanup() throws Exception, KeeperException {
124     for (HTableDescriptor table : ADMIN.listTables()) {
125       ADMIN.disableTable(table.getTableName());
126       deleteTable(table.getTableName());
127     }
128     for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129       if (ns.getName().startsWith(prefix)) {
130         ADMIN.deleteNamespace(ns.getName());
131       }
132     }
133     assertTrue("Quota manager not initialized", UTIL.getHBaseCluster().getMaster()
134         .getMasterQuotaManager().isQuotaInitialized());
135   }
136 
137   @Test
138   public void testTableOperations() throws Exception {
139     String nsp = prefix + "_np2";
140     NamespaceDescriptor nspDesc =
141         NamespaceDescriptor.create(nsp)
142             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144     ADMIN.createNamespace(nspDesc);
145     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146     assertEquals(ADMIN.listNamespaces().length, 3);
147     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
148     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
149 
150     HTableDescriptor tableDescOne =
151         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
152     tableDescOne.addFamily(fam1);
153     HTableDescriptor tableDescTwo =
154         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
155     tableDescTwo.addFamily(fam1);
156     HTableDescriptor tableDescThree =
157         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
158     tableDescThree.addFamily(fam1);
159     ADMIN.createTable(tableDescOne);
160     boolean constraintViolated = false;
161     try {
162       ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
163     } catch (Exception exp) {
164       assertTrue(exp instanceof IOException);
165       constraintViolated = true;
166     } finally {
167       assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
168         constraintViolated);
169     }
170     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
171     NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
172     assertNotNull(nspState);
173     assertTrue(nspState.getTables().size() == 2);
174     assertTrue(nspState.getRegionCount() == 5);
175     constraintViolated = false;
176     try {
177       ADMIN.createTable(tableDescThree);
178     } catch (Exception exp) {
179       assertTrue(exp instanceof IOException);
180       constraintViolated = true;
181     } finally {
182       assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
183         constraintViolated);
184     }
185   }
186 
187   @Test
188   public void testValidQuotas() throws Exception {
189     boolean exceptionCaught = false;
190     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
191     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
192     NamespaceDescriptor nspDesc =
193         NamespaceDescriptor.create(prefix + "vq1")
194             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
195             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
196     try {
197       ADMIN.createNamespace(nspDesc);
198     } catch (Exception exp) {
199       LOG.warn(exp);
200       exceptionCaught = true;
201     } finally {
202       assertTrue(exceptionCaught);
203       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
204     }
205     nspDesc =
206         NamespaceDescriptor.create(prefix + "vq2")
207             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
208             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
209     try {
210       ADMIN.createNamespace(nspDesc);
211     } catch (Exception exp) {
212       LOG.warn(exp);
213       exceptionCaught = true;
214     } finally {
215       assertTrue(exceptionCaught);
216       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
217     }
218     nspDesc =
219         NamespaceDescriptor.create(prefix + "vq3")
220             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
221             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
222     try {
223       ADMIN.createNamespace(nspDesc);
224     } catch (Exception exp) {
225       LOG.warn(exp);
226       exceptionCaught = true;
227     } finally {
228       assertTrue(exceptionCaught);
229       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
230     }
231     nspDesc =
232         NamespaceDescriptor.create(prefix + "vq4")
233             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
234             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
235     try {
236       ADMIN.createNamespace(nspDesc);
237     } catch (Exception exp) {
238       LOG.warn(exp);
239       exceptionCaught = true;
240     } finally {
241       assertTrue(exceptionCaught);
242       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
243     }
244   }
245 
246   @Test
247   public void testDeleteTable() throws Exception {
248     String namespace = prefix + "_dummy";
249     NamespaceDescriptor nspDesc =
250         NamespaceDescriptor.create(namespace)
251             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
252             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
253     ADMIN.createNamespace(nspDesc);
254     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
255     NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
256     assertNotNull("Namespace state found null for " + namespace, stateInfo);
257     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
258     HTableDescriptor tableDescOne =
259         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
260     tableDescOne.addFamily(fam1);
261     HTableDescriptor tableDescTwo =
262         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
263     tableDescTwo.addFamily(fam1);
264     ADMIN.createTable(tableDescOne);
265     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
266     stateInfo = getNamespaceState(nspDesc.getName());
267     assertNotNull("Namespace state found to be null.", stateInfo);
268     assertEquals(2, stateInfo.getTables().size());
269     assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
270     assertEquals(6, stateInfo.getRegionCount());
271     ADMIN.disableTable(tableDescOne.getTableName());
272     deleteTable(tableDescOne.getTableName());
273     stateInfo = getNamespaceState(nspDesc.getName());
274     assertNotNull("Namespace state found to be null.", stateInfo);
275     assertEquals(5, stateInfo.getRegionCount());
276     assertEquals(1, stateInfo.getTables().size());
277     ADMIN.disableTable(tableDescTwo.getTableName());
278     deleteTable(tableDescTwo.getTableName());
279     ADMIN.deleteNamespace(namespace);
280     stateInfo = getNamespaceState(namespace);
281     assertNull("Namespace state not found to be null.", stateInfo);
282   }
283 
284   public static class CPRegionServerObserver extends BaseRegionServerObserver {
285     private volatile boolean shouldFailMerge = false;
286 
287     public void failMerge(boolean fail) {
288       shouldFailMerge = fail;
289     }
290 
291     private boolean triggered = false;
292 
293     public synchronized void waitUtilTriggered() throws InterruptedException {
294       while (!triggered) {
295         wait();
296       }
297     }
298 
299     @Override
300     public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
301         Region regionA, Region regionB) throws IOException {
302       triggered = true;
303       notifyAll();
304       if (shouldFailMerge) {
305         throw new IOException("fail merge");
306       }
307     }
308   }
309 
310   @Test
311   public void testRegionMerge() throws Exception {
312     String nsp1 = prefix + "_regiontest";
313     NamespaceDescriptor nspDesc =
314         NamespaceDescriptor.create(nsp1)
315             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
316             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
317     ADMIN.createNamespace(nspDesc);
318     final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
319     byte[] columnFamily = Bytes.toBytes("info");
320     HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
321     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
322     final int initialRegions = 3;
323     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
324     try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
325         Table table = connection.getTable(tableTwo)) {
326       UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
327     }
328     ADMIN.flush(tableTwo);
329     List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
330     Collections.sort(hris);
331     // merge the two regions
332     final Set<String> encodedRegionNamesToMerge =
333         Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
334     ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
335       false);
336     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
337     hris = ADMIN.getTableRegions(tableTwo);
338     assertEquals(initialRegions - 1, hris.size());
339     Collections.sort(hris);
340 
341     final HRegionInfo hriToSplit = hris.get(1);
342     ADMIN.split(tableTwo, Bytes.toBytes("500"));
343 
344     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
345 
346       @Override
347       public boolean evaluate() throws Exception {
348         RegionStates regionStates =
349             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
350         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
351           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
352             return false;
353           }
354           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
355             return false;
356           }
357         }
358         return true;
359       }
360 
361       @Override
362       public String explainFailure() throws Exception {
363         RegionStates regionStates =
364             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
365         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
366           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
367             return hriToSplit + " which is expected to be split is still online";
368           }
369           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
370             return hri + " is still in not opened";
371           }
372         }
373         return "Unknown";
374       }
375     });
376     hris = ADMIN.getTableRegions(tableTwo);
377     assertEquals(initialRegions, hris.size());
378     Collections.sort(hris);
379 
380     // fail region merge through Coprocessor hook
381     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
382     HRegionServer regionServer = cluster.getRegionServer(0);
383     RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
384     Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
385     CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
386     regionServerObserver.failMerge(true);
387     regionServerObserver.triggered = false;
388 
389     ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
390       false);
391     regionServerObserver.waitUtilTriggered();
392     hris = ADMIN.getTableRegions(tableTwo);
393     assertEquals(initialRegions, hris.size());
394     Collections.sort(hris);
395     // verify that we cannot split
396     ADMIN.split(tableTwo, Bytes.toBytes("6"));
397     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
398     assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
399   }
400 
401   private void waitForMergeToComplete(final TableName tableTwo,
402       final Set<String> encodedRegionNamesToMerge) throws Exception {
403     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
404 
405       @Override
406       public boolean evaluate() throws Exception {
407         RegionStates regionStates =
408             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
409         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
410           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
411             return false;
412           }
413           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
414             return false;
415           }
416         }
417         return true;
418       }
419 
420       @Override
421       public String explainFailure() throws Exception {
422         RegionStates regionStates =
423             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
424         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
425           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
426             return hri + " which is expected to be merged is still online";
427           }
428           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
429             return hri + " is still in not opened";
430           }
431         }
432         return "Unknown";
433       }
434     });
435   }
436 
437   @Ignore("Hangs on occasion waiting on countdown latch") @Test
438   public void testRegionOperations() throws Exception {
439     String nsp1 = prefix + "_regiontest";
440     NamespaceDescriptor nspDesc =
441         NamespaceDescriptor.create(nsp1)
442             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
443             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
444     ADMIN.createNamespace(nspDesc);
445     boolean constraintViolated = false;
446     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
447     byte[] columnFamily = Bytes.toBytes("info");
448     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
449     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
450     NamespaceTableAndRegionInfo stateInfo;
451     try {
452       ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
453     } catch (Exception exp) {
454       assertTrue(exp instanceof DoNotRetryIOException);
455       LOG.info(exp);
456       constraintViolated = true;
457     } finally {
458       assertTrue(constraintViolated);
459     }
460     assertFalse(ADMIN.tableExists(tableOne));
461     // This call will pass.
462     ADMIN.createTable(tableDescOne);
463     Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
464     HTable htable = (HTable) connection.getTable(tableOne);
465     UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
466     ADMIN.flush(tableOne);
467     stateInfo = getNamespaceState(nsp1);
468     assertEquals(1, stateInfo.getTables().size());
469     assertEquals(1, stateInfo.getRegionCount());
470     restartMaster();
471     ADMIN.split(tableOne, Bytes.toBytes("500"));
472     HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473     CustomObserver observer =
474         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475           CustomObserver.class.getName());
476     assertNotNull(observer);
477     observer.postSplit.await();
478     assertEquals(2, ADMIN.getTableRegions(tableOne).size());
479     actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
480     observer =
481         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
482           CustomObserver.class.getName());
483     assertNotNull(observer);
484     ADMIN.split(
485       tableOne,
486       getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
487           .getEndKey()));
488     observer.postSplit.await();
489     // Make sure no regions have been added.
490     List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
491     assertEquals(2, hris.size());
492     assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
493 
494     htable.close();
495   }
496 
497   /*
498    * Create a table and make sure that the table creation fails after adding this table entry into
499    * namespace quota cache. Now correct the failure and recreate the table with same name.
500    * HBASE-13394
501    */
502   @Test
503   public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
504     String nsp1 = prefix + "_testRecreateTable";
505     NamespaceDescriptor nspDesc =
506         NamespaceDescriptor.create(nsp1)
507             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
508             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
509     ADMIN.createNamespace(nspDesc);
510     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
511     byte[] columnFamily = Bytes.toBytes("info");
512     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
513     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
514     MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
515     try {
516       try {
517         ADMIN.createTable(tableDescOne);
518         fail("Table " + tableOne.toString() + "creation should fail.");
519       } catch (Exception exp) {
520         LOG.error(exp);
521       }
522       assertFalse(ADMIN.tableExists(tableOne));
523 
524       NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
525       assertEquals("First table creation failed in namespace so number of tables in namespace "
526           + "should be 0.", 0, nstate.getTables().size());
527 
528       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
529       try {
530         ADMIN.createTable(tableDescOne);
531       } catch (Exception e) {
532         fail("Table " + tableOne.toString() + "creation should succeed.");
533         LOG.error(e);
534       }
535       assertTrue(ADMIN.tableExists(tableOne));
536       nstate = getNamespaceState(nsp1);
537       assertEquals("First table was created successfully so table size in namespace should "
538           + "be one now.", 1, nstate.getTables().size());
539     } finally {
540       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
541       if (ADMIN.tableExists(tableOne)) {
542         ADMIN.disableTable(tableOne);
543         deleteTable(tableOne);
544       }
545       ADMIN.deleteNamespace(nsp1);
546     }
547   }
548 
549   private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
550       IOException {
551     return getQuotaManager().getState(namespace);
552   }
553 
554   byte[] getSplitKey(byte[] startKey, byte[] endKey) {
555     String skey = Bytes.toString(startKey);
556     int key;
557     if (StringUtils.isBlank(skey)) {
558       key = Integer.parseInt(Bytes.toString(endKey)) / 2;
559     } else {
560       key = (int) (Integer.parseInt(skey) * 1.5);
561     }
562     return Bytes.toBytes("" + key);
563   }
564 
565   public static class CustomObserver extends BaseRegionObserver {
566     volatile CountDownLatch postSplit;
567     volatile CountDownLatch preSplitBeforePONR;
568 
569     @Override
570     public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
571         throws IOException {
572       postSplit.countDown();
573     }
574 
575     @Override
576     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
577         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
578       preSplitBeforePONR.countDown();
579     }
580 
581     @Override
582     public void start(CoprocessorEnvironment e) throws IOException {
583       postSplit = new CountDownLatch(1);
584       preSplitBeforePONR = new CountDownLatch(1);
585     }
586   }
587 
588   @Test
589   public void testStatePreserve() throws Exception {
590     final String nsp1 = prefix + "_testStatePreserve";
591     NamespaceDescriptor nspDesc =
592         NamespaceDescriptor.create(nsp1)
593             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
594             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
595     ADMIN.createNamespace(nspDesc);
596     TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
597     TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
598     TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
599     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
600     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
601     tableDescOne.addFamily(fam1);
602     HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
603     tableDescTwo.addFamily(fam1);
604     HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
605     tableDescThree.addFamily(fam1);
606     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
607     ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
608     ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
609     ADMIN.disableTable(tableThree);
610     deleteTable(tableThree);
611     // wait for chore to complete
612     UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
613       @Override
614       public boolean evaluate() throws Exception {
615         return (getNamespaceState(nsp1).getTables().size() == 2);
616       }
617     });
618     NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
619     restartMaster();
620     NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
621     assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
622         .getTables().size(), after.getTables().size());
623   }
624 
625   private static void waitForQuotaEnabled() throws Exception {
626     UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
627       @Override
628       public boolean evaluate() throws Exception {
629         HMaster master = UTIL.getHBaseCluster().getMaster();
630         if (master == null) {
631           return false;
632         }
633         MasterQuotaManager quotaManager = master.getMasterQuotaManager();
634         return quotaManager != null && quotaManager.isQuotaInitialized();
635       }
636     });
637   }
638 
639   private void restartMaster() throws Exception {
640     UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
641     UTIL.getHBaseCluster().waitOnMaster(0);
642     UTIL.getHBaseCluster().startMaster();
643     waitForQuotaEnabled();
644   }
645 
646   private NamespaceAuditor getQuotaManager() {
647     return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
648   }
649 
650   public static class MasterSyncObserver extends BaseMasterObserver {
651     volatile CountDownLatch tableDeletionLatch;
652     static boolean throwExceptionInPreCreateTableHandler;
653 
654     @Override
655     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
656         TableName tableName) throws IOException {
657       tableDeletionLatch = new CountDownLatch(1);
658     }
659 
660     @Override
661     public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
662         TableName tableName) throws IOException {
663       tableDeletionLatch.countDown();
664     }
665 
666     @Override
667     public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
668         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
669       if (throwExceptionInPreCreateTableHandler) {
670         throw new IOException("Throw exception as it is demanded.");
671       }
672     }
673   }
674 
675   private void deleteTable(final TableName tableName) throws Exception {
676     // NOTE: We need a latch because admin is not sync,
677     // so the postOp coprocessor method may be called after the admin operation returned.
678     MasterSyncObserver observer =
679         (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
680             .findCoprocessor(MasterSyncObserver.class.getName());
681     ADMIN.deleteTable(tableName);
682     observer.tableDeletionLatch.await();
683   }
684 
685   @Test(expected = QuotaExceededException.class)
686   public void testExceedTableQuotaInNamespace() throws Exception {
687     String nsp = prefix + "_testExceedTableQuotaInNamespace";
688     NamespaceDescriptor nspDesc =
689         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
690             .build();
691     ADMIN.createNamespace(nspDesc);
692     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
693     assertEquals(ADMIN.listNamespaces().length, 3);
694     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
695     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
696     HTableDescriptor tableDescOne =
697         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
698     tableDescOne.addFamily(fam1);
699     HTableDescriptor tableDescTwo =
700         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
701     tableDescTwo.addFamily(fam1);
702     ADMIN.createTable(tableDescOne);
703     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
704   }
705   
706   @Test(expected = QuotaExceededException.class)
707   public void testCloneSnapshotQuotaExceed() throws Exception {
708     String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
709     NamespaceDescriptor nspDesc =
710         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
711             .build();
712     ADMIN.createNamespace(nspDesc);
713     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
714     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
715     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
716     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
717     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
718     tableDescOne.addFamily(fam1);
719     ADMIN.createTable(tableDescOne);
720     String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
721     ADMIN.snapshot(snapshot, tableName);
722     ADMIN.cloneSnapshot(snapshot, cloneTableName);
723     ADMIN.deleteSnapshot(snapshot);
724   }
725 
726   @Test
727   public void testCloneSnapshot() throws Exception {
728     String nsp = prefix + "_testCloneSnapshot";
729     NamespaceDescriptor nspDesc =
730         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
731             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
732     ADMIN.createNamespace(nspDesc);
733     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
734     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
735     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
736 
737     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
738     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
739     tableDescOne.addFamily(fam1);
740 
741     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
742     String snapshot = "snapshot_testCloneSnapshot";
743     ADMIN.snapshot(snapshot, tableName);
744     ADMIN.cloneSnapshot(snapshot, cloneTableName);
745 
746     int tableLength;
747     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
748       tableLength = locator.getStartKeys().length;
749     }
750     assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
751 
752     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
753       tableLength = locator.getStartKeys().length;
754     }
755     assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
756 
757     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
758     assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
759     assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
760 
761     ADMIN.deleteSnapshot(snapshot);
762   }
763 
764   @Test
765   public void testRestoreSnapshot() throws Exception {
766     String nsp = prefix + "_testRestoreSnapshot";
767     NamespaceDescriptor nspDesc =
768         NamespaceDescriptor.create(nsp)
769             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
770     ADMIN.createNamespace(nspDesc);
771     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
772     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
773     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
774     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
775     tableDescOne.addFamily(fam1);
776     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
777 
778     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
779     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
780 
781     String snapshot = "snapshot_testRestoreSnapshot";
782     ADMIN.snapshot(snapshot, tableName1);
783 
784     List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
785     Collections.sort(regions);
786 
787     ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
788     Thread.sleep(2000);
789     assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
790 
791     ADMIN.disableTable(tableName1);
792     ADMIN.restoreSnapshot(snapshot);
793 
794     assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
795 
796     ADMIN.enableTable(tableName1);
797     ADMIN.deleteSnapshot(snapshot);
798   }
799 
800   @Test
801   public void testRestoreSnapshotQuotaExceed() throws Exception {
802     String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
803     NamespaceDescriptor nspDesc =
804         NamespaceDescriptor.create(nsp)
805             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
806     ADMIN.createNamespace(nspDesc);
807     NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
808     assertNotNull("Namespace descriptor found null.", ndesc);
809     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
810     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
811     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
812     tableDescOne.addFamily(fam1);
813 
814     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
815 
816     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
817     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
818 
819     String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
820     // snapshot has 4 regions
821     ADMIN.snapshot(snapshot, tableName1);
822     // recreate table with 1 region and set max regions to 3 for namespace
823     ADMIN.disableTable(tableName1);
824     ADMIN.deleteTable(tableName1);
825     ADMIN.createTable(tableDescOne);
826     ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
827     ADMIN.modifyNamespace(ndesc);
828 
829     ADMIN.disableTable(tableName1);
830     try {
831       ADMIN.restoreSnapshot(snapshot);
832       fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
833           + " wraps IOException into RestoreSnapshotException");
834     } catch (RestoreSnapshotException ignore) {
835       assertTrue(ignore.getCause() instanceof QuotaExceededException);
836     }
837     assertEquals(1, getNamespaceState(nsp).getRegionCount());
838     ADMIN.enableTable(tableName1);
839     ADMIN.deleteSnapshot(snapshot);
840   }
841 }