1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.ArrayList;
26 import java.util.HashSet;
27 import java.util.Map;
28 import java.util.concurrent.ConcurrentHashMap;
29 import java.util.concurrent.atomic.AtomicBoolean;
30 import java.util.concurrent.atomic.AtomicInteger;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.HBaseConfiguration;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.master.TableLockManager;
38 import org.apache.hadoop.hbase.procedure2.Procedure;
39 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure;
40 import org.apache.hadoop.hbase.testclassification.MasterTests;
41 import org.apache.hadoop.hbase.testclassification.SmallTests;
42 import org.junit.After;
43 import org.junit.Before;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47 import static org.junit.Assert.assertEquals;
48 import static org.junit.Assert.assertFalse;
49 import static org.junit.Assert.assertTrue;
50 import static org.junit.Assert.fail;
51
52 @Category({MasterTests.class, SmallTests.class})
53 public class TestMasterProcedureScheduler {
54 private static final Log LOG = LogFactory.getLog(TestMasterProcedureScheduler.class);
55
56 private MasterProcedureScheduler queue;
57 private Configuration conf;
58
59 @Before
60 public void setUp() throws IOException {
61 conf = HBaseConfiguration.create();
62 queue = new MasterProcedureScheduler(conf, new TableLockManager.NullTableLockManager());
63 }
64
65 @After
66 public void tearDown() throws IOException {
67 assertEquals(0, queue.size());
68 }
69
70 @Test
71 public void testConcurrentCreateDelete() throws Exception {
72 final MasterProcedureScheduler procQueue = queue;
73 final TableName table = TableName.valueOf("testtb");
74 final AtomicBoolean running = new AtomicBoolean(true);
75 final AtomicBoolean failure = new AtomicBoolean(false);
76 Thread createThread = new Thread() {
77 @Override
78 public void run() {
79 try {
80 TestTableProcedure proc = new TestTableProcedure(1, table,
81 TableProcedureInterface.TableOperationType.CREATE);
82 while (running.get() && !failure.get()) {
83 if (procQueue.tryAcquireTableExclusiveLock(proc, table)) {
84 procQueue.releaseTableExclusiveLock(proc, table);
85 }
86 }
87 } catch (Throwable e) {
88 LOG.error("create failed", e);
89 failure.set(true);
90 }
91 }
92 };
93
94 Thread deleteThread = new Thread() {
95 @Override
96 public void run() {
97 try {
98 TestTableProcedure proc = new TestTableProcedure(2, table,
99 TableProcedureInterface.TableOperationType.DELETE);
100 while (running.get() && !failure.get()) {
101 if (procQueue.tryAcquireTableExclusiveLock(proc, table)) {
102 procQueue.releaseTableExclusiveLock(proc, table);
103 }
104 procQueue.markTableAsDeleted(table);
105 }
106 } catch (Throwable e) {
107 LOG.error("delete failed", e);
108 failure.set(true);
109 }
110 }
111 };
112
113 createThread.start();
114 deleteThread.start();
115 for (int i = 0; i < 100 && running.get() && !failure.get(); ++i) {
116 Thread.sleep(100);
117 }
118 running.set(false);
119 createThread.join();
120 deleteThread.join();
121 assertEquals(false, failure.get());
122 }
123
124
125
126
127 @Test
128 public void testSimpleTableOpsQueues() throws Exception {
129 final int NUM_TABLES = 10;
130 final int NUM_ITEMS = 10;
131
132 int count = 0;
133 for (int i = 1; i <= NUM_TABLES; ++i) {
134 TableName tableName = TableName.valueOf(String.format("test-%04d", i));
135
136 for (int j = 1; j <= NUM_ITEMS; ++j) {
137 queue.addBack(new TestTableProcedure(i * 1000 + j, tableName,
138 TableProcedureInterface.TableOperationType.EDIT));
139 assertEquals(++count, queue.size());
140 }
141 }
142 assertEquals(NUM_TABLES * NUM_ITEMS, queue.size());
143
144 for (int j = 1; j <= NUM_ITEMS; ++j) {
145 for (int i = 1; i <= NUM_TABLES; ++i) {
146 Procedure proc = queue.poll();
147 assertTrue(proc != null);
148 TableName tableName = ((TestTableProcedure)proc).getTableName();
149 queue.tryAcquireTableExclusiveLock(proc, tableName);
150 queue.releaseTableExclusiveLock(proc, tableName);
151 queue.completionCleanup(proc);
152 assertEquals(--count, queue.size());
153 assertEquals(i * 1000 + j, proc.getProcId());
154 }
155 }
156 assertEquals(0, queue.size());
157
158 for (int i = 1; i <= NUM_TABLES; ++i) {
159 TableName tableName = TableName.valueOf(String.format("test-%04d", i));
160
161 assertTrue(queue.markTableAsDeleted(tableName));
162 }
163 }
164
165
166
167
168
169 @Test
170 public void testCreateDeleteTableOperationsWithWriteLock() throws Exception {
171 TableName tableName = TableName.valueOf("testtb");
172
173 queue.addBack(new TestTableProcedure(1, tableName,
174 TableProcedureInterface.TableOperationType.EDIT));
175
176
177 assertFalse(queue.markTableAsDeleted(tableName));
178
179
180 Procedure proc = queue.poll();
181 assertEquals(1, proc.getProcId());
182
183 assertTrue(queue.tryAcquireTableExclusiveLock(proc, tableName));
184
185 assertEquals(0, queue.size());
186 assertFalse(queue.markTableAsDeleted(tableName));
187
188 queue.releaseTableExclusiveLock(proc, tableName);
189
190 assertTrue(queue.markTableAsDeleted(tableName));
191 }
192
193
194
195
196
197 @Test
198 public void testCreateDeleteTableOperationsWithReadLock() throws Exception {
199 final TableName tableName = TableName.valueOf("testtb");
200 final int nitems = 2;
201
202 for (int i = 1; i <= nitems; ++i) {
203 queue.addBack(new TestTableProcedure(i, tableName,
204 TableProcedureInterface.TableOperationType.READ));
205 }
206
207
208 assertFalse(queue.markTableAsDeleted(tableName));
209
210 Procedure[] procs = new Procedure[nitems];
211 for (int i = 0; i < nitems; ++i) {
212
213 Procedure proc = procs[i] = queue.poll();
214 assertEquals(i + 1, proc.getProcId());
215
216 assertTrue(queue.tryAcquireTableSharedLock(proc, tableName));
217
218 assertFalse(queue.markTableAsDeleted(tableName));
219 }
220
221 for (int i = 0; i < nitems; ++i) {
222
223 assertFalse(queue.markTableAsDeleted(tableName));
224
225 queue.releaseTableSharedLock(procs[i], tableName);
226 }
227
228
229 assertEquals(0, queue.size());
230
231 assertTrue(queue.markTableAsDeleted(tableName));
232 }
233
234
235
236
237 @Test
238 public void testVerifyRwLocks() throws Exception {
239 TableName tableName = TableName.valueOf("testtb");
240 queue.addBack(new TestTableProcedure(1, tableName,
241 TableProcedureInterface.TableOperationType.EDIT));
242 queue.addBack(new TestTableProcedure(2, tableName,
243 TableProcedureInterface.TableOperationType.READ));
244 queue.addBack(new TestTableProcedure(3, tableName,
245 TableProcedureInterface.TableOperationType.EDIT));
246 queue.addBack(new TestTableProcedure(4, tableName,
247 TableProcedureInterface.TableOperationType.READ));
248 queue.addBack(new TestTableProcedure(5, tableName,
249 TableProcedureInterface.TableOperationType.READ));
250
251
252 Procedure proc = queue.poll();
253 assertEquals(1, proc.getProcId());
254 assertEquals(true, queue.tryAcquireTableExclusiveLock(proc, tableName));
255
256
257 assertEquals(null, queue.poll(0));
258
259
260 queue.releaseTableExclusiveLock(proc, tableName);
261
262
263 Procedure rdProc = queue.poll();
264 assertEquals(2, rdProc.getProcId());
265 assertEquals(true, queue.tryAcquireTableSharedLock(rdProc, tableName));
266
267
268 Procedure wrProc = queue.poll();
269 assertEquals(3, wrProc.getProcId());
270 assertEquals(false, queue.tryAcquireTableExclusiveLock(wrProc, tableName));
271
272
273 queue.releaseTableSharedLock(rdProc, tableName);
274 assertEquals(true, queue.tryAcquireTableExclusiveLock(wrProc, tableName));
275
276
277 assertEquals(null, queue.poll(0));
278
279
280 queue.releaseTableExclusiveLock(wrProc, tableName);
281
282
283 rdProc = queue.poll();
284 assertEquals(4, rdProc.getProcId());
285 assertEquals(true, queue.tryAcquireTableSharedLock(rdProc, tableName));
286
287
288 Procedure rdProc2 = queue.poll();
289 assertEquals(5, rdProc2.getProcId());
290 assertEquals(true, queue.tryAcquireTableSharedLock(rdProc2, tableName));
291
292
293 queue.releaseTableSharedLock(rdProc, tableName);
294 queue.releaseTableSharedLock(rdProc2, tableName);
295
296
297 assertEquals(0, queue.size());
298 assertTrue("queue should be deleted", queue.markTableAsDeleted(tableName));
299 }
300
301 @Test
302 public void testVerifyNamespaceRwLocks() throws Exception {
303 String nsName1 = "ns1";
304 String nsName2 = "ns2";
305 TableName tableName1 = TableName.valueOf(nsName1, "testtb");
306 TableName tableName2 = TableName.valueOf(nsName2, "testtb");
307 queue.addBack(new TestNamespaceProcedure(1, nsName1,
308 TableProcedureInterface.TableOperationType.EDIT));
309 queue.addBack(new TestTableProcedure(2, tableName1,
310 TableProcedureInterface.TableOperationType.EDIT));
311 queue.addBack(new TestTableProcedure(3, tableName2,
312 TableProcedureInterface.TableOperationType.EDIT));
313 queue.addBack(new TestNamespaceProcedure(4, nsName2,
314 TableProcedureInterface.TableOperationType.EDIT));
315
316
317 Procedure procNs1 = queue.poll();
318 assertEquals(1, procNs1.getProcId());
319 assertEquals(true, queue.tryAcquireNamespaceExclusiveLock(procNs1, nsName1));
320
321
322 Procedure procNs2 = queue.poll();
323 assertEquals(4, procNs2.getProcId());
324 assertEquals(true, queue.tryAcquireNamespaceExclusiveLock(procNs2, nsName2));
325 queue.releaseNamespaceExclusiveLock(procNs2, nsName2);
326 queue.yield(procNs2);
327
328
329 procNs2 = queue.poll();
330 assertEquals(3, procNs2.getProcId());
331 assertEquals(true, queue.tryAcquireTableExclusiveLock(procNs2, tableName2));
332
333
334 Procedure procNs2b = queue.poll();
335 assertEquals(4, procNs2b.getProcId());
336 assertEquals(false, queue.tryAcquireNamespaceExclusiveLock(procNs2b, nsName2));
337 queue.yield(procNs2b);
338
339
340 queue.releaseNamespaceExclusiveLock(procNs1, nsName1);
341
342
343 long procId = queue.poll().getProcId();
344 assertEquals(2, procId);
345
346 queue.releaseTableExclusiveLock(procNs2, tableName2);
347
348
349 procId = queue.poll().getProcId();
350 assertEquals(4, procId);
351 }
352
353
354
355
356
357 @Test(timeout=90000)
358 public void testConcurrentWriteOps() throws Exception {
359 final TestTableProcSet procSet = new TestTableProcSet(queue);
360
361 final int NUM_ITEMS = 10;
362 final int NUM_TABLES = 4;
363 final AtomicInteger opsCount = new AtomicInteger(0);
364 for (int i = 0; i < NUM_TABLES; ++i) {
365 TableName tableName = TableName.valueOf(String.format("testtb-%04d", i));
366 for (int j = 1; j < NUM_ITEMS; ++j) {
367 procSet.addBack(new TestTableProcedure(i * 100 + j, tableName,
368 TableProcedureInterface.TableOperationType.EDIT));
369 opsCount.incrementAndGet();
370 }
371 }
372 assertEquals(opsCount.get(), queue.size());
373
374 final Thread[] threads = new Thread[NUM_TABLES * 2];
375 final HashSet<TableName> concurrentTables = new HashSet<TableName>();
376 final ArrayList<String> failures = new ArrayList<String>();
377 final AtomicInteger concurrentCount = new AtomicInteger(0);
378 for (int i = 0; i < threads.length; ++i) {
379 threads[i] = new Thread() {
380 @Override
381 public void run() {
382 while (opsCount.get() > 0) {
383 try {
384 Procedure proc = procSet.acquire();
385 if (proc == null) {
386 queue.signalAll();
387 if (opsCount.get() > 0) {
388 continue;
389 }
390 break;
391 }
392
393 TableName tableId = procSet.getTableName(proc);
394 synchronized (concurrentTables) {
395 assertTrue("unexpected concurrency on " + tableId, concurrentTables.add(tableId));
396 }
397 assertTrue(opsCount.decrementAndGet() >= 0);
398 try {
399 long procId = proc.getProcId();
400 int concurrent = concurrentCount.incrementAndGet();
401 assertTrue("inc-concurrent="+ concurrent +" 1 <= concurrent <= "+ NUM_TABLES,
402 concurrent >= 1 && concurrent <= NUM_TABLES);
403 LOG.debug("[S] tableId="+ tableId +" procId="+ procId +" concurrent="+ concurrent);
404 Thread.sleep(2000);
405 concurrent = concurrentCount.decrementAndGet();
406 LOG.debug("[E] tableId="+ tableId +" procId="+ procId +" concurrent="+ concurrent);
407 assertTrue("dec-concurrent=" + concurrent, concurrent < NUM_TABLES);
408 } finally {
409 synchronized (concurrentTables) {
410 assertTrue(concurrentTables.remove(tableId));
411 }
412 procSet.release(proc);
413 }
414 } catch (Throwable e) {
415 LOG.error("Failed " + e.getMessage(), e);
416 synchronized (failures) {
417 failures.add(e.getMessage());
418 }
419 } finally {
420 queue.signalAll();
421 }
422 }
423 }
424 };
425 threads[i].start();
426 }
427 for (int i = 0; i < threads.length; ++i) {
428 threads[i].join();
429 }
430 assertTrue(failures.toString(), failures.isEmpty());
431 assertEquals(0, opsCount.get());
432 assertEquals(0, queue.size());
433
434 for (int i = 1; i <= NUM_TABLES; ++i) {
435 TableName table = TableName.valueOf(String.format("testtb-%04d", i));
436 assertTrue("queue should be deleted, table=" + table, queue.markTableAsDeleted(table));
437 }
438 }
439
440 public static class TestTableProcSet {
441 private final MasterProcedureScheduler queue;
442
443 public TestTableProcSet(final MasterProcedureScheduler queue) {
444 this.queue = queue;
445 }
446
447 public void addBack(Procedure proc) {
448 queue.addBack(proc);
449 }
450
451 public void addFront(Procedure proc) {
452 queue.addFront(proc);
453 }
454
455 public Procedure acquire() {
456 Procedure proc = null;
457 boolean avail = false;
458 while (!avail) {
459 proc = queue.poll();
460 if (proc == null) break;
461 switch (getTableOperationType(proc)) {
462 case CREATE:
463 case DELETE:
464 case EDIT:
465 avail = queue.tryAcquireTableExclusiveLock(proc, getTableName(proc));
466 break;
467 case READ:
468 avail = queue.tryAcquireTableSharedLock(proc, getTableName(proc));
469 break;
470 default:
471 throw new UnsupportedOperationException();
472 }
473 if (!avail) {
474 addFront(proc);
475 LOG.debug("yield procId=" + proc);
476 }
477 }
478 return proc;
479 }
480
481 public void release(Procedure proc) {
482 switch (getTableOperationType(proc)) {
483 case CREATE:
484 case DELETE:
485 case EDIT:
486 queue.releaseTableExclusiveLock(proc, getTableName(proc));
487 break;
488 case READ:
489 queue.releaseTableSharedLock(proc, getTableName(proc));
490 break;
491 default:
492 break;
493 }
494 }
495
496 public TableName getTableName(Procedure proc) {
497 return ((TableProcedureInterface)proc).getTableName();
498 }
499
500 public TableProcedureInterface.TableOperationType getTableOperationType(Procedure proc) {
501 return ((TableProcedureInterface)proc).getTableOperationType();
502 }
503 }
504
505 public static class TestTableProcedure extends TestProcedure
506 implements TableProcedureInterface {
507 private final TableOperationType opType;
508 private final TableName tableName;
509
510 public TestTableProcedure() {
511 throw new UnsupportedOperationException("recovery should not be triggered here");
512 }
513
514 public TestTableProcedure(long procId, TableName tableName, TableOperationType opType) {
515 super(procId);
516 this.tableName = tableName;
517 this.opType = opType;
518 }
519
520 @Override
521 public TableName getTableName() {
522 return tableName;
523 }
524
525 @Override
526 public TableOperationType getTableOperationType() {
527 return opType;
528 }
529 }
530
531 public static class TestNamespaceProcedure extends TestProcedure
532 implements TableProcedureInterface {
533 private final TableOperationType opType;
534 private final String nsName;
535
536 public TestNamespaceProcedure() {
537 throw new UnsupportedOperationException("recovery should not be triggered here");
538 }
539
540 public TestNamespaceProcedure(long procId, String nsName, TableOperationType opType) {
541 super(procId);
542 this.nsName = nsName;
543 this.opType = opType;
544 }
545
546 @Override
547 public TableName getTableName() {
548 return TableName.NAMESPACE_TABLE_NAME;
549 }
550
551 @Override
552 public TableOperationType getTableOperationType() {
553 return opType;
554 }
555 }
556 }