1
2
3
4
5
6
7
8
9
10
11
12 package org.apache.hadoop.hbase.quotas;
13
14 import static org.junit.Assert.assertEquals;
15 import static org.junit.Assert.assertNull;
16 import static org.junit.Assert.assertTrue;
17 import static org.junit.Assert.fail;
18
19 import com.google.common.collect.Iterables;
20
21 import java.util.concurrent.TimeUnit;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.Cell;
26 import org.apache.hadoop.hbase.CellScanner;
27 import org.apache.hadoop.hbase.HBaseTestingUtility;
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.client.Admin;
31 import org.apache.hadoop.hbase.client.Result;
32 import org.apache.hadoop.hbase.client.ResultScanner;
33 import org.apache.hadoop.hbase.client.Scan;
34 import org.apache.hadoop.hbase.client.Table;
35 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
36 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
37 import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
38 import org.apache.hadoop.hbase.security.User;
39 import org.apache.hadoop.hbase.testclassification.MediumTests;
40 import org.junit.AfterClass;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45
46
47
48 @Category({ MediumTests.class })
49 public class TestQuotaAdmin {
50 private static final Log LOG = LogFactory.getLog(TestQuotaAdmin.class);
51
52 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
53
54 @BeforeClass
55 public static void setUpBeforeClass() throws Exception {
56 TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
57 TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000);
58 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
59 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
60 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
61 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
62 TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
63 TEST_UTIL.startMiniCluster(1);
64 TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
65 }
66
67 @AfterClass
68 public static void tearDownAfterClass() throws Exception {
69 TEST_UTIL.shutdownMiniCluster();
70 }
71
72 @Test
73 public void testThrottleType() throws Exception {
74 Admin admin = TEST_UTIL.getHBaseAdmin();
75 String userName = User.getCurrent().getShortName();
76
77 admin.setQuota(QuotaSettingsFactory
78 .throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
79 admin.setQuota(QuotaSettingsFactory
80 .throttleUser(userName, ThrottleType.WRITE_NUMBER, 12, TimeUnit.MINUTES));
81 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
82
83 try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
84 int countThrottle = 0;
85 int countGlobalBypass = 0;
86 for (QuotaSettings settings: scanner) {
87 switch (settings.getQuotaType()) {
88 case THROTTLE:
89 ThrottleSettings throttle = (ThrottleSettings)settings;
90 if (throttle.getSoftLimit() == 6) {
91 assertEquals(ThrottleType.READ_NUMBER, throttle.getThrottleType());
92 } else if (throttle.getSoftLimit() == 12) {
93 assertEquals(ThrottleType.WRITE_NUMBER, throttle.getThrottleType());
94 } else {
95 fail("should not come here, because don't set quota with this limit");
96 }
97 assertEquals(userName, throttle.getUserName());
98 assertEquals(null, throttle.getTableName());
99 assertEquals(null, throttle.getNamespace());
100 assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
101 countThrottle++;
102 break;
103 case GLOBAL_BYPASS:
104 countGlobalBypass++;
105 break;
106 default:
107 fail("unexpected settings type: " + settings.getQuotaType());
108 }
109 }
110 assertEquals(2, countThrottle);
111 assertEquals(1, countGlobalBypass);
112 }
113
114 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
115 assertNumResults(1, null);
116 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
117 assertNumResults(0, null);
118 }
119
120 @Test
121 public void testSimpleScan() throws Exception {
122 Admin admin = TEST_UTIL.getHBaseAdmin();
123 String userName = User.getCurrent().getShortName();
124
125 admin.setQuota(QuotaSettingsFactory.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6,
126 TimeUnit.MINUTES));
127 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
128
129 try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
130 int countThrottle = 0;
131 int countGlobalBypass = 0;
132 for (QuotaSettings settings : scanner) {
133 LOG.debug(settings);
134 switch (settings.getQuotaType()) {
135 case THROTTLE:
136 ThrottleSettings throttle = (ThrottleSettings) settings;
137 assertEquals(userName, throttle.getUserName());
138 assertEquals(null, throttle.getTableName());
139 assertEquals(null, throttle.getNamespace());
140 assertEquals(6, throttle.getSoftLimit());
141 assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
142 countThrottle++;
143 break;
144 case GLOBAL_BYPASS:
145 countGlobalBypass++;
146 break;
147 default:
148 fail("unexpected settings type: " + settings.getQuotaType());
149 }
150 }
151 assertEquals(1, countThrottle);
152 assertEquals(1, countGlobalBypass);
153 }
154
155 admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
156 assertNumResults(1, null);
157 admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
158 assertNumResults(0, null);
159 }
160
161 @Test
162 public void testQuotaRetrieverFilter() throws Exception {
163 Admin admin = TEST_UTIL.getHBaseAdmin();
164 TableName[] tables =
165 new TableName[] { TableName.valueOf("T0"), TableName.valueOf("T01"),
166 TableName.valueOf("NS0:T2"), };
167 String[] namespaces = new String[] { "NS0", "NS01", "NS2" };
168 String[] users = new String[] { "User0", "User01", "User2" };
169
170 for (String user : users) {
171 admin.setQuota(QuotaSettingsFactory.throttleUser(user, ThrottleType.REQUEST_NUMBER, 1,
172 TimeUnit.MINUTES));
173
174 for (TableName table : tables) {
175 admin.setQuota(QuotaSettingsFactory.throttleUser(user, table, ThrottleType.REQUEST_NUMBER,
176 2, TimeUnit.MINUTES));
177 }
178
179 for (String ns : namespaces) {
180 admin.setQuota(QuotaSettingsFactory.throttleUser(user, ns, ThrottleType.REQUEST_NUMBER, 3,
181 TimeUnit.MINUTES));
182 }
183 }
184 assertNumResults(21, null);
185
186 for (TableName table : tables) {
187 admin.setQuota(QuotaSettingsFactory.throttleTable(table, ThrottleType.REQUEST_NUMBER, 4,
188 TimeUnit.MINUTES));
189 }
190 assertNumResults(24, null);
191
192 for (String ns : namespaces) {
193 admin.setQuota(QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_NUMBER, 5,
194 TimeUnit.MINUTES));
195 }
196 assertNumResults(27, null);
197
198 assertNumResults(7, new QuotaFilter().setUserFilter("User0"));
199 assertNumResults(0, new QuotaFilter().setUserFilter("User"));
200 assertNumResults(21, new QuotaFilter().setUserFilter("User.*"));
201 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0"));
202 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setTableFilter("NS.*"));
203 assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setTableFilter("T"));
204 assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T.*"));
205 assertNumResults(3, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS0"));
206 assertNumResults(0, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS"));
207 assertNumResults(9, new QuotaFilter().setUserFilter("User.*").setNamespaceFilter("NS.*"));
208 assertNumResults(6, new QuotaFilter().setUserFilter("User.*").setTableFilter("T0")
209 .setNamespaceFilter("NS0"));
210 assertNumResults(1, new QuotaFilter().setTableFilter("T0"));
211 assertNumResults(0, new QuotaFilter().setTableFilter("T"));
212 assertNumResults(2, new QuotaFilter().setTableFilter("T.*"));
213 assertNumResults(3, new QuotaFilter().setTableFilter(".*T.*"));
214 assertNumResults(1, new QuotaFilter().setNamespaceFilter("NS0"));
215 assertNumResults(0, new QuotaFilter().setNamespaceFilter("NS"));
216 assertNumResults(3, new QuotaFilter().setNamespaceFilter("NS.*"));
217
218 for (String user : users) {
219 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user));
220 for (TableName table : tables) {
221 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, table));
222 }
223 for (String ns : namespaces) {
224 admin.setQuota(QuotaSettingsFactory.unthrottleUser(user, ns));
225 }
226 }
227 assertNumResults(6, null);
228
229 for (TableName table : tables) {
230 admin.setQuota(QuotaSettingsFactory.unthrottleTable(table));
231 }
232 assertNumResults(3, null);
233
234 for (String ns : namespaces) {
235 admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(ns));
236 }
237 assertNumResults(0, null);
238 }
239
240 private void assertNumResults(int expected, final QuotaFilter filter) throws Exception {
241 assertEquals(expected, countResults(filter));
242 }
243
244 @Test
245 public void testSetGetRemoveRPCQuota() throws Exception {
246 testSetGetRemoveRPCQuota(ThrottleType.REQUEST_SIZE);
247 testSetGetRemoveRPCQuota(ThrottleType.REQUEST_CAPACITY_UNIT);
248 }
249
250 private void testSetGetRemoveRPCQuota(ThrottleType throttleType) throws Exception {
251 Admin admin = TEST_UTIL.getHBaseAdmin();
252 final TableName tn = TableName.valueOf("sq_table1");
253 QuotaSettings settings =
254 QuotaSettingsFactory.throttleTable(tn, throttleType, 2L, TimeUnit.HOURS);
255 admin.setQuota(settings);
256
257
258 verifyRecordPresentInQuotaTable(throttleType, 2L, TimeUnit.HOURS);
259
260
261 verifyFetchableViaAPI(admin, throttleType, 2L, TimeUnit.HOURS);
262
263
264 QuotaSettings removeQuota = QuotaSettingsFactory.unthrottleTable(tn);
265 admin.setQuota(removeQuota);
266
267
268 verifyRecordNotPresentInQuotaTable();
269
270
271 verifyNotFetchableViaAPI(admin);
272 }
273
274 @Test
275 public void testSetModifyRemoveRPCQuota() throws Exception {
276 Admin admin = TEST_UTIL.getHBaseAdmin();
277 final TableName tn = TableName.valueOf("sq_table1");
278 QuotaSettings settings =
279 QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
280 admin.setQuota(settings);
281
282
283 verifyRecordPresentInQuotaTable(ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
284
285
286 verifyFetchableViaAPI(admin, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
287
288
289 QuotaSettings newSettings =
290 QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS);
291 admin.setQuota(newSettings);
292
293
294 verifyRecordPresentInQuotaTable(ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS);
295
296
297 verifyFetchableViaAPI(admin, ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS);
298
299
300 QuotaSettings removeQuota = QuotaSettingsFactory.unthrottleTable(tn);
301 admin.setQuota(removeQuota);
302
303
304 verifyRecordNotPresentInQuotaTable();
305
306
307 verifyNotFetchableViaAPI(admin);
308
309 }
310
311 private void verifyRecordPresentInQuotaTable(ThrottleType type, long limit, TimeUnit tu)
312 throws Exception {
313
314 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
315 ResultScanner scanner = quotaTable.getScanner(new Scan())) {
316 Result r = Iterables.getOnlyElement(scanner);
317 CellScanner cells = r.cellScanner();
318 assertTrue("Expected to find a cell", cells.advance());
319 assertRPCQuota(type, limit, tu, cells.current());
320 }
321 }
322
323 private void verifyRecordNotPresentInQuotaTable() throws Exception {
324
325 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
326 ResultScanner scanner = quotaTable.getScanner(new Scan())) {
327 assertNull("Did not expect to find a quota entry", scanner.next());
328 }
329 }
330
331 private void verifyFetchableViaAPI(Admin admin, ThrottleType type, long limit, TimeUnit tu)
332 throws Exception {
333
334 try (QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration())) {
335 assertRPCQuota(type, limit, tu, Iterables.getOnlyElement(quotaScanner));
336 }
337 }
338
339 private void verifyNotFetchableViaAPI(Admin admin) throws Exception {
340
341 try (QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration())) {
342 assertNull("Did not expect to find a quota entry", quotaScanner.next());
343 }
344 }
345
346 private void assertRPCQuota(ThrottleType type, long limit, TimeUnit tu, Cell cell)
347 throws Exception {
348 Quotas q = QuotaTableUtil.quotasFromData(cell.getValue());
349 assertTrue("Quota should have rpc quota defined", q.hasThrottle());
350
351 QuotaProtos.Throttle rpcQuota = q.getThrottle();
352 QuotaProtos.TimedQuota t = null;
353
354 switch (type) {
355 case REQUEST_SIZE:
356 assertTrue(rpcQuota.hasReqSize());
357 t = rpcQuota.getReqSize();
358 break;
359 case READ_NUMBER:
360 assertTrue(rpcQuota.hasReadNum());
361 t = rpcQuota.getReadNum();
362 break;
363 case READ_SIZE:
364 assertTrue(rpcQuota.hasReadSize());
365 t = rpcQuota.getReadSize();
366 break;
367 case REQUEST_NUMBER:
368 assertTrue(rpcQuota.hasReqNum());
369 t = rpcQuota.getReqNum();
370 break;
371 case WRITE_NUMBER:
372 assertTrue(rpcQuota.hasWriteNum());
373 t = rpcQuota.getWriteNum();
374 break;
375 case WRITE_SIZE:
376 assertTrue(rpcQuota.hasWriteSize());
377 t = rpcQuota.getWriteSize();
378 break;
379 case REQUEST_CAPACITY_UNIT:
380 assertTrue(rpcQuota.hasReqCapacityUnit());
381 t = rpcQuota.getReqCapacityUnit();
382 break;
383 case READ_CAPACITY_UNIT:
384 assertTrue(rpcQuota.hasReadCapacityUnit());
385 t = rpcQuota.getReadCapacityUnit();
386 break;
387 case WRITE_CAPACITY_UNIT:
388 assertTrue(rpcQuota.hasWriteCapacityUnit());
389 t = rpcQuota.getWriteCapacityUnit();
390 break;
391 default:
392 }
393
394 assertEquals(t.getSoftLimit(), limit);
395 assertEquals(t.getTimeUnit(), ProtobufUtil.toProtoTimeUnit(tu));
396 }
397
398 private void assertRPCQuota(ThrottleType type, long limit, TimeUnit tu,
399 QuotaSettings actualSettings) throws Exception {
400 assertTrue(
401 "The actual QuotaSettings was not an instance of " + ThrottleSettings.class + " but of "
402 + actualSettings.getClass(), actualSettings instanceof ThrottleSettings);
403 QuotaProtos.ThrottleRequest throttleRequest = ((ThrottleSettings) actualSettings).getProto();
404 assertEquals(limit, throttleRequest.getTimedQuota().getSoftLimit());
405 assertEquals(ProtobufUtil.toProtoTimeUnit(tu), throttleRequest.getTimedQuota().getTimeUnit());
406 assertEquals(ProtobufUtil.toProtoThrottleType(type), throttleRequest.getType());
407 }
408
409 private int countResults(final QuotaFilter filter) throws Exception {
410 QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration(), filter);
411 try {
412 int count = 0;
413 for (QuotaSettings settings : scanner) {
414 LOG.debug(settings);
415 count++;
416 }
417 return count;
418 } finally {
419 scanner.close();
420 }
421 }
422 }