View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  
12  package org.apache.hadoop.hbase.quotas;
13  
14  import static org.junit.Assert.assertEquals;
15  
16  import java.util.concurrent.TimeUnit;
17  
18  import org.apache.commons.logging.Log;
19  import org.apache.commons.logging.LogFactory;
20  import org.apache.hadoop.hbase.HBaseTestingUtility;
21  import org.apache.hadoop.hbase.HConstants;
22  import org.apache.hadoop.hbase.TableName;
23  import org.apache.hadoop.hbase.client.Admin;
24  import org.apache.hadoop.hbase.client.Get;
25  import org.apache.hadoop.hbase.client.HTable;
26  import org.apache.hadoop.hbase.client.Put;
27  import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
28  import org.apache.hadoop.hbase.client.Table;
29  import org.apache.hadoop.hbase.security.User;
30  import org.apache.hadoop.hbase.testclassification.MediumTests;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
33  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
34  import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
35  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
36  import org.junit.After;
37  import org.junit.AfterClass;
38  import org.junit.BeforeClass;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  @Category({ MediumTests.class })
43  public class TestQuotaThrottle {
44    final Log LOG = LogFactory.getLog(getClass());
45  
46    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47  
48    private final static byte[] FAMILY = Bytes.toBytes("cf");
49    private final static byte[] QUALIFIER = Bytes.toBytes("q");
50  
51    private final static TableName[] TABLE_NAMES = new TableName[] {
52        TableName.valueOf("TestQuotaAdmin0"), TableName.valueOf("TestQuotaAdmin1"),
53        TableName.valueOf("TestQuotaAdmin2") };
54  
55    private static HTable[] tables;
56  
57    @BeforeClass
58    public static void setUpBeforeClass() throws Exception {
59      TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
60      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
61      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
62      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
63      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
64      TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
65      TEST_UTIL.getConfiguration().setBoolean(
66        QuotaUtil.QUOTA_RETRYABLE_THROTTING_EXCEPTION_CONF_KEY, false);
67      TEST_UTIL.startMiniCluster(1);
68      TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
69      QuotaCache.setTEST_FORCE_REFRESH(true);
70  
71      tables = new HTable[TABLE_NAMES.length];
72      for (int i = 0; i < TABLE_NAMES.length; ++i) {
73        tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY);
74      }
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      for (int i = 0; i < tables.length; ++i) {
80        if (tables[i] != null) {
81          tables[i].close();
82          TEST_UTIL.deleteTable(TABLE_NAMES[i]);
83        }
84      }
85  
86      TEST_UTIL.shutdownMiniCluster();
87    }
88  
89    @After
90    public void tearDown() throws Exception {
91      for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
92        RegionServerQuotaManager quotaManager = rst.getRegionServer().getRegionServerQuotaManager();
93        QuotaCache quotaCache = quotaManager.getQuotaCache();
94        quotaCache.getNamespaceQuotaCache().clear();
95        quotaCache.getTableQuotaCache().clear();
96        quotaCache.getUserQuotaCache().clear();
97      }
98    }
99  
100   @Test(timeout = 60000)
101   public void testUserGlobalThrottle() throws Exception {
102     final Admin admin = TEST_UTIL.getHBaseAdmin();
103     final String userName = User.getCurrent().getShortName();
104 
105     // Add 6req/min limit
106     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
107       TimeUnit.MINUTES));
108     triggerUserCacheRefresh(false, TABLE_NAMES);
109 
110     // should execute at max 6 requests
111     assertEquals(6, doPuts(100, tables));
112 
113     // wait a minute and you should get other 6 requests executed
114     waitMinuteQuota();
115     assertEquals(6, doPuts(100, tables));
116 
117     // Remove all the limits
118     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
119     triggerUserCacheRefresh(true, TABLE_NAMES);
120     assertEquals(60, doPuts(60, tables));
121     assertEquals(60, doGets(60, tables));
122   }
123 
124   @Test(timeout=60000)
125   public void testUserGlobalReadAndWriteThrottle() throws Exception {
126     final Admin admin = TEST_UTIL.getHBaseAdmin();
127     final String userName = User.getCurrent().getShortName();
128 
129     // Add 6req/min limit for read request
130     admin.setQuota(QuotaSettingsFactory
131       .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
132     triggerUserCacheRefresh(false, TABLE_NAMES);
133 
134     // not limit for write request and should execute at max 6 read requests
135     assertEquals(60, doPuts(60, tables));
136     assertEquals(6, doGets(100, tables));
137 
138     waitMinuteQuota();
139 
140     // Add 6req/min limit for write request
141     admin.setQuota(QuotaSettingsFactory
142       .throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
143     triggerUserCacheRefresh(false, TABLE_NAMES);
144 
145     // should execute at max 6 read requests and at max 6 write write requests
146     assertEquals(6, doGets(100, tables));
147     assertEquals(6, doPuts(60, tables));
148 
149     // Remove all the limits
150     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
151     triggerUserCacheRefresh(true, TABLE_NAMES);
152     assertEquals(60, doPuts(60, tables));
153     assertEquals(60, doGets(60, tables));
154   }
155 
156   @Test(timeout = 60000)
157   public void testUserTableThrottle() throws Exception {
158     final Admin admin = TEST_UTIL.getHBaseAdmin();
159     final String userName = User.getCurrent().getShortName();
160 
161     // Add 6req/min limit
162     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
163       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
164     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
165 
166     // should execute at max 6 requests on tables[0] and have no limit on tables[1]
167     assertEquals(6, doPuts(100, tables[0]));
168     assertEquals(30, doPuts(30, tables[1]));
169 
170     // wait a minute and you should get other 6 requests executed
171     waitMinuteQuota();
172     assertEquals(6, doPuts(100, tables[0]));
173 
174     // Remove all the limits
175     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
176     triggerUserCacheRefresh(true, TABLE_NAMES);
177     assertEquals(60, doPuts(60, tables));
178     assertEquals(60, doGets(60, tables));
179   }
180 
181   @Test(timeout=60000)
182   public void testUserTableReadAndWriteThrottle() throws Exception {
183     final Admin admin = TEST_UTIL.getHBaseAdmin();
184     final String userName = User.getCurrent().getShortName();
185 
186     // Add 6req/min limit for write request on tables[0]
187     admin.setQuota(QuotaSettingsFactory
188       .throttleUser(userName, TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
189     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
190 
191     // should execute at max 6 write requests and have no limit for read request
192     assertEquals(6, doPuts(100, tables[0]));
193     assertEquals(60, doGets(60, tables[0]));
194 
195     // no limit on tables[1]
196     assertEquals(60, doPuts(60, tables[1]));
197     assertEquals(60, doGets(60, tables[1]));
198 
199     // wait a minute and you should get other 6  write requests executed
200     waitMinuteQuota();
201 
202     // Add 6req/min limit for read request on tables[0]
203     admin.setQuota(QuotaSettingsFactory
204       .throttleUser(userName, TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
205     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
206 
207     // should execute at max 6 read requests and at max 6 write requests
208     assertEquals(6, doPuts(100, tables[0]));
209     assertEquals(6, doGets(60, tables[0]));
210 
211     // no limit on tables[1]
212     assertEquals(30, doPuts(30, tables[1]));
213     assertEquals(30, doGets(30, tables[1]));
214 
215     // Remove all the limits
216     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
217     triggerUserCacheRefresh(true, TABLE_NAMES);
218     assertEquals(60, doPuts(60, tables));
219     assertEquals(60, doGets(60, tables));
220   }
221 
222   @Test(timeout = 60000)
223   public void testUserNamespaceThrottle() throws Exception {
224     final Admin admin = TEST_UTIL.getHBaseAdmin();
225     final String userName = User.getCurrent().getShortName();
226     final String NAMESPACE = "default";
227 
228     // Add 6req/min limit
229     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, NAMESPACE,
230       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
231     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
232 
233     // should execute at max 6 requests on tables[0] and have no limit on tables[1]
234     assertEquals(6, doPuts(100, tables[0]));
235 
236     // wait a minute and you should get other 6 requests executed
237     waitMinuteQuota();
238     assertEquals(6, doPuts(100, tables[1]));
239 
240     // Remove all the limits
241     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
242     triggerUserCacheRefresh(true, TABLE_NAMES);
243     assertEquals(60, doPuts(60, tables));
244     assertEquals(60, doGets(60, tables));
245   }
246 
247   @Test(timeout=60000)
248   public void testUserNamespaceReadAndWriteThrottle() throws Exception {
249     final Admin admin = TEST_UTIL.getHBaseAdmin();
250     final String userName = User.getCurrent().getShortName();
251     final String NAMESPACE = "default";
252 
253     // Add 6req/min limit for read request
254     admin.setQuota(QuotaSettingsFactory
255       .throttleUser(userName, NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
256     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
257 
258     // should execute at max 6 read requests and have no limit for write request
259     assertEquals(6, doGets(60, tables[0]));
260     assertEquals(60, doPuts(60, tables[0]));
261 
262     waitMinuteQuota();
263 
264     // Add 6req/min limit for write request, too
265     admin.setQuota(QuotaSettingsFactory
266       .throttleUser(userName, NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
267     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
268 
269     // should execute at max 6 read requests and at max 6 write requests
270     assertEquals(6, doGets(60, tables[0]));
271     assertEquals(6, doPuts(60, tables[0]));
272 
273     // Remove all the limits
274     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
275     triggerUserCacheRefresh(true, TABLE_NAMES);
276     assertEquals(60, doPuts(60, tables));
277     assertEquals(60, doGets(60, tables));
278   }
279 
280   @Test(timeout = 60000)
281   public void testTableGlobalThrottle() throws Exception {
282     final Admin admin = TEST_UTIL.getHBaseAdmin();
283 
284     // Add 6req/min limit
285     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
286       6, TimeUnit.MINUTES));
287     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
288 
289     // should execute at max 6 requests
290     assertEquals(6, doPuts(100, tables[0]));
291     // should have no limits
292     assertEquals(30, doPuts(30, tables[1]));
293 
294     // wait a minute and you should get other 6 requests executed
295     waitMinuteQuota();
296     assertEquals(6, doPuts(100, tables[0]));
297 
298     // Remove all the limits
299     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
300     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
301     assertEquals(80, doGets(80, tables[0], tables[1]));
302   }
303 
304   @Test(timeout=60000)
305   public void testTableGlobalReadAndWriteThrottle() throws Exception {
306     final Admin admin = TEST_UTIL.getHBaseAdmin();
307 
308     // Add 6req/min limit for read request
309     admin.setQuota(QuotaSettingsFactory
310       .throttleTable(TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
311     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
312 
313     // should execute at max 6 read requests and have no limit for write request
314     assertEquals(6, doGets(100, tables[0]));
315     assertEquals(100, doPuts(100, tables[0]));
316     // should have no limits on tables[1]
317     assertEquals(30, doPuts(30, tables[1]));
318     assertEquals(30, doGets(30, tables[1]));
319 
320     // wait a minute and you should get other 6 requests executed
321     waitMinuteQuota();
322 
323     // Add 6req/min limit for write request, too
324     admin.setQuota(QuotaSettingsFactory
325       .throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
326     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
327 
328     // should execute at max 6 read requests and at max 6 write requests
329     assertEquals(6, doGets(100, tables[0]));
330     assertEquals(6, doPuts(100, tables[0]));
331     // should have no limits on tables[1]
332     assertEquals(30, doPuts(30, tables[1]));
333     assertEquals(30, doGets(30, tables[1]));
334 
335     // Remove all the limits
336     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
337     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
338     assertEquals(80, doGets(80, tables[0], tables[1]));
339   }
340 
341   @Test(timeout = 60000)
342   public void testNamespaceGlobalThrottle() throws Exception {
343     final Admin admin = TEST_UTIL.getHBaseAdmin();
344     final String NAMESPACE = "default";
345 
346     // Add 6req/min limit
347     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
348       6, TimeUnit.MINUTES));
349     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
350 
351     // should execute at max 6 requests
352     assertEquals(6, doPuts(100, tables[0]));
353 
354     // wait a minute and you should get other 6 requests executed
355     waitMinuteQuota();
356     assertEquals(6, doPuts(100, tables[1]));
357 
358     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
359     triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
360     assertEquals(40, doPuts(40, tables[0]));
361   }
362 
363   @Test(timeout=60000)
364   public void testNamespaceGlobalReadAndWriteThrottle() throws Exception {
365     final Admin admin = TEST_UTIL.getHBaseAdmin();
366     final String NAMESPACE = "default";
367 
368     // Add 6req/min limit for write request
369     admin.setQuota(QuotaSettingsFactory
370       .throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
371     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
372 
373     // should execute at max 6 write requests and no limit for read request
374     assertEquals(6, doPuts(100, tables[0]));
375     assertEquals(100, doGets(100, tables[0]));
376 
377     // wait a minute and you should get other 6 requests executed
378     waitMinuteQuota();
379 
380     // Add 6req/min limit for read request, too
381     admin.setQuota(QuotaSettingsFactory
382       .throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
383     triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
384 
385     // should execute at max 6 write requests and at max 6 read requests
386     assertEquals(6, doPuts(100, tables[0]));
387     assertEquals(6, doGets(100, tables[0]));
388 
389     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
390     triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
391     assertEquals(40, doPuts(40, tables[0]));
392   }
393 
394   @Test(timeout = 60000)
395   public void testUserAndTableThrottle() throws Exception {
396     final Admin admin = TEST_UTIL.getHBaseAdmin();
397     final String userName = User.getCurrent().getShortName();
398 
399     // Add 6req/min limit for the user on tables[0]
400     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[0],
401       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
402     triggerUserCacheRefresh(false, TABLE_NAMES[0]);
403     // Add 12req/min limit for the user
404     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 12,
405       TimeUnit.MINUTES));
406     triggerUserCacheRefresh(false, TABLE_NAMES[1], TABLE_NAMES[2]);
407     // Add 8req/min limit for the tables[1]
408     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[1], ThrottleType.REQUEST_NUMBER,
409       8, TimeUnit.MINUTES));
410     triggerTableCacheRefresh(false, TABLE_NAMES[1]);
411     // Add a lower table level throttle on tables[0]
412     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
413       3, TimeUnit.MINUTES));
414     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
415 
416     // should execute at max 12 requests
417     assertEquals(12, doGets(100, tables[2]));
418 
419     // should execute at max 8 requests
420     waitMinuteQuota();
421     assertEquals(8, doGets(100, tables[1]));
422 
423     // should execute at max 3 requests
424     waitMinuteQuota();
425     assertEquals(3, doPuts(100, tables[0]));
426 
427     // Remove all the throttling rules
428     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
429     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
430     triggerUserCacheRefresh(true, TABLE_NAMES[0], TABLE_NAMES[1]);
431 
432     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[1]));
433     triggerTableCacheRefresh(true, TABLE_NAMES[1]);
434     waitMinuteQuota();
435     assertEquals(40, doGets(40, tables[1]));
436 
437     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
438     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
439     waitMinuteQuota();
440     assertEquals(40, doGets(40, tables[0]));
441   }
442 
443   @Test(timeout = 60000)
444   public void testUserGlobalBypassThrottle() throws Exception {
445     final Admin admin = TEST_UTIL.getHBaseAdmin();
446     final String userName = User.getCurrent().getShortName();
447     final String NAMESPACE = "default";
448 
449     // Add 6req/min limit for tables[0]
450     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.REQUEST_NUMBER,
451       6, TimeUnit.MINUTES));
452     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
453     // Add 13req/min limit for the user
454     admin.setQuota(QuotaSettingsFactory.throttleNamespace(NAMESPACE, ThrottleType.REQUEST_NUMBER,
455       13, TimeUnit.MINUTES));
456     triggerNamespaceCacheRefresh(false, TABLE_NAMES[1]);
457 
458     // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
459     assertEquals(6, doPuts(100, tables[0]));
460     assertEquals(7, doGets(100, tables[1]));
461     waitMinuteQuota();
462 
463     // Set the global bypass for the user
464     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
465     admin.setQuota(QuotaSettingsFactory.throttleUser(userName, TABLE_NAMES[2],
466       ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
467     triggerUserCacheRefresh(false, TABLE_NAMES[2]);
468     assertEquals(30, doGets(30, tables[0]));
469     assertEquals(30, doGets(30, tables[1]));
470     waitMinuteQuota();
471 
472     // Remove the global bypass
473     // should execute at max 6 requests on table[0] and (13 - 6) on table[1]
474     admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
475     admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[2]));
476     triggerUserCacheRefresh(true, TABLE_NAMES[2]);
477     assertEquals(6, doPuts(100, tables[0]));
478     assertEquals(7, doGets(100, tables[1]));
479 
480     // unset throttle
481     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
482     admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
483     waitMinuteQuota();
484     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
485     triggerNamespaceCacheRefresh(true, TABLE_NAMES[1]);
486     assertEquals(30, doGets(30, tables[0]));
487     assertEquals(30, doGets(30, tables[1]));
488   }
489 
490   @Test
491   public void testTableExistsGetThrottle() throws Exception {
492     final Admin admin = TEST_UTIL.getHBaseAdmin();
493 
494     // Add throttle quota
495     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0],
496         ThrottleType.REQUEST_NUMBER, 100, TimeUnit.MINUTES));
497     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
498 
499     Table table = TEST_UTIL.getConnection().getTable(TABLE_NAMES[0]);
500     // An exists call when having throttle quota
501     table.exists(new Get(Bytes.toBytes("abc")));
502 
503     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
504     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
505   }
506 
507   @Test
508   public void testTableWriteCapacityUnitThrottle() throws Exception {
509     final Admin admin = TEST_UTIL.getHBaseAdmin();
510 
511     // Add 6CU/min limit
512     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0],
513       ThrottleType.WRITE_CAPACITY_UNIT, 6, TimeUnit.MINUTES));
514     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
515 
516     // should execute at max 6 capacity units because each put size is 1 capacity unit
517     assertEquals(6, doPuts(20, 10, tables[0]));
518 
519     // wait a minute and you should execute at max 3 capacity units because each put size is 2
520     // capacity unit
521     waitMinuteQuota();
522     assertEquals(3, doPuts(20, 1025, tables[0]));
523 
524     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
525     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
526   }
527 
528   @Test
529   public void testTableReadCapacityUnitThrottle() throws Exception {
530     final Admin admin = TEST_UTIL.getHBaseAdmin();
531 
532     // Add 6CU/min limit
533     admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0],
534       ThrottleType.READ_CAPACITY_UNIT, 6, TimeUnit.MINUTES));
535     triggerTableCacheRefresh(false, TABLE_NAMES[0]);
536 
537     assertEquals(20, doPuts(20, 10, tables[0]));
538     // should execute at max 6 capacity units because each get size is 1 capacity unit
539     assertEquals(6, doGets(20, tables[0]));
540 
541     assertEquals(20, doPuts(20, 2015, tables[0]));
542     // wait a minute and you should execute at max 3 capacity units because each get size is 2
543     // capacity unit on tables[0]
544     waitMinuteQuota();
545     assertEquals(3, doGets(20, tables[0]));
546 
547     admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
548     triggerTableCacheRefresh(true, TABLE_NAMES[0]);
549   }
550 
551   private int doPuts(int maxOps, final HTable... tables) throws Exception {
552     return doPuts(maxOps, -1, tables);
553   }
554 
555   private int doPuts(int maxOps, int valueSize, final HTable... tables) throws Exception {
556     int count = 0;
557     try {
558       while (count < maxOps) {
559         Put put = new Put(Bytes.toBytes("row-" + count));
560         byte[] value;
561         if (valueSize < 0) {
562           value = Bytes.toBytes("data-" + count);
563         } else {
564           value = generateValue(valueSize);
565         }
566         put.addColumn(FAMILY, QUALIFIER, value);
567         for (final Table table : tables) {
568           table.put(put);
569         }
570         count += tables.length;
571       }
572     } catch (RetriesExhaustedWithDetailsException e) {
573       for (Throwable t : e.getCauses()) {
574         if (!((t instanceof ThrottlingException))||((t instanceof RpcThrottlingException))) {
575           throw e;
576         }
577       }
578       LOG.error("put failed after nRetries=" + count, e);
579     }
580     return count;
581   }
582 
583   private byte[] generateValue(int valueSize) {
584     byte[] bytes = new byte[valueSize];
585     for (int i = 0; i < valueSize; i++) {
586       bytes[i] = 'a';
587     }
588     return bytes;
589   }
590 
591   private long doGets(int maxOps, final HTable... tables) throws Exception {
592     int count = 0;
593     try {
594       while (count < maxOps) {
595         Get get = new Get(Bytes.toBytes("row-" + count));
596         for (final HTable table : tables) {
597           table.get(get);
598         }
599         count += tables.length;
600       }
601     } catch (ThrottlingException|RpcThrottlingException e) {
602       LOG.error("get failed after nRetries=" + count, e);
603     }
604     return count;
605   }
606 
607   private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception {
608     triggerCacheRefresh(bypass, true, false, false, tables);
609   }
610 
611   private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception {
612     triggerCacheRefresh(bypass, false, true, false, tables);
613   }
614 
615   private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception {
616     triggerCacheRefresh(bypass, false, false, true, tables);
617   }
618 
619   private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter,
620       boolean nsLimiter, final TableName... tables) throws Exception {
621     for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
622       RegionServerQuotaManager quotaManager = rst.getRegionServer().getRegionServerQuotaManager();
623       QuotaCache quotaCache = quotaManager.getQuotaCache();
624 
625       quotaCache.triggerCacheRefresh();
626       Thread.sleep(250);
627 
628       for (TableName table : tables) {
629         quotaCache.getTableLimiter(table);
630       }
631 
632       boolean isUpdated = false;
633       while (!isUpdated) {
634         isUpdated = true;
635         for (TableName table : tables) {
636           boolean isBypass = true;
637           if (userLimiter) {
638             isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
639           }
640           if (tableLimiter) {
641             isBypass &= quotaCache.getTableLimiter(table).isBypass();
642           }
643           if (nsLimiter) {
644             isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
645           }
646           if (isBypass != bypass) {
647             isUpdated = false;
648             Thread.sleep(250);
649             break;
650           }
651         }
652       }
653 
654       LOG.debug("QuotaCache");
655       LOG.debug(quotaCache.getNamespaceQuotaCache());
656       LOG.debug(quotaCache.getTableQuotaCache());
657       LOG.debug(quotaCache.getUserQuotaCache());
658     }
659   }
660 
661   private void waitMinuteQuota() {
662     EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge(
663         EnvironmentEdgeManager.currentTime() + 70000));
664   }
665 }