1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.coprocessor;
21
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertNotNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.lang.reflect.Method;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.List;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.Cell;
39 import org.apache.hadoop.hbase.CellUtil;
40 import org.apache.hadoop.hbase.Coprocessor;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.KeyValue;
46 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
47 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
48 import org.apache.hadoop.hbase.testclassification.MediumTests;
49 import org.apache.hadoop.hbase.MiniHBaseCluster;
50 import org.apache.hadoop.hbase.ServerName;
51 import org.apache.hadoop.hbase.TableName;
52 import org.apache.hadoop.hbase.client.Admin;
53 import org.apache.hadoop.hbase.client.Append;
54 import org.apache.hadoop.hbase.client.Delete;
55 import org.apache.hadoop.hbase.client.Durability;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.client.HTable;
58 import org.apache.hadoop.hbase.client.Increment;
59 import org.apache.hadoop.hbase.client.Put;
60 import org.apache.hadoop.hbase.client.Result;
61 import org.apache.hadoop.hbase.client.ResultScanner;
62 import org.apache.hadoop.hbase.client.RowMutations;
63 import org.apache.hadoop.hbase.client.Scan;
64 import org.apache.hadoop.hbase.client.Table;
65 import org.apache.hadoop.hbase.filter.FilterAllFilter;
66 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
67 import org.apache.hadoop.hbase.io.hfile.HFile;
68 import org.apache.hadoop.hbase.io.hfile.HFileContext;
69 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
70 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
71 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
72 import org.apache.hadoop.hbase.regionserver.HRegion;
73 import org.apache.hadoop.hbase.regionserver.InternalScanner;
74 import org.apache.hadoop.hbase.regionserver.NoLimitScannerContext;
75 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
76 import org.apache.hadoop.hbase.regionserver.ScanType;
77 import org.apache.hadoop.hbase.regionserver.ScannerContext;
78 import org.apache.hadoop.hbase.regionserver.Store;
79 import org.apache.hadoop.hbase.regionserver.StoreFile;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
82 import org.apache.hadoop.hbase.util.JVMClusterUtil;
83 import org.apache.hadoop.hbase.util.Threads;
84 import org.apache.hadoop.hbase.wal.WALKey;
85 import org.junit.AfterClass;
86 import org.junit.Assert;
87 import org.junit.BeforeClass;
88 import org.junit.Ignore;
89 import org.junit.Test;
90 import org.junit.experimental.categories.Category;
91 import org.mockito.Mockito;
92
93 @Category(MediumTests.class)
94 public class TestRegionObserverInterface {
95 private static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
96
97 public static final TableName TEST_TABLE = TableName.valueOf("TestTable");
98 public final static byte[] A = Bytes.toBytes("a");
99 public final static byte[] B = Bytes.toBytes("b");
100 public final static byte[] C = Bytes.toBytes("c");
101 public final static byte[] ROW = Bytes.toBytes("testrow");
102 public final static byte[] FAMILY = Bytes.toBytes("f");
103
104 private static HBaseTestingUtility util = new HBaseTestingUtility();
105 private static MiniHBaseCluster cluster = null;
106
107 @BeforeClass
108 public static void setupBeforeClass() throws Exception {
109
110 Configuration conf = util.getConfiguration();
111 conf.setBoolean("hbase.master.distributed.log.replay", true);
112 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
113 "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver",
114 "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver$Legacy");
115
116 util.startMiniCluster();
117 cluster = util.getMiniHBaseCluster();
118 }
119
120 @AfterClass
121 public static void tearDownAfterClass() throws Exception {
122 util.shutdownMiniCluster();
123 }
124
125 @Test (timeout=300000)
126 public void testRegionObserver() throws IOException {
127 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRegionObserver");
128
129
130 Table table = util.createTable(tableName, new byte[][] {A, B, C});
131 try {
132 verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet",
133 "hadPrePut", "hadPostPut", "hadDelete", "hadPostStartRegionOperation",
134 "hadPostCloseRegionOperation", "hadPostBatchMutateIndispensably" }, tableName,
135 new Boolean[] { false, false, false, false, false, false, false, false });
136
137 Put put = new Put(ROW);
138 put.add(A, A, A);
139 put.add(B, B, B);
140 put.add(C, C, C);
141 table.put(put);
142
143 verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet",
144 "hadPrePut", "hadPostPut", "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete",
145 "hadPostStartRegionOperation", "hadPostCloseRegionOperation",
146 "hadPostBatchMutateIndispensably" }, TEST_TABLE, new Boolean[] { false, false, true,
147 true, true, true, false, true, true, true });
148
149 verifyMethodResult(SimpleRegionObserver.class,
150 new String[] {"getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose"},
151 tableName,
152 new Integer[] {1, 1, 0, 0});
153
154 Get get = new Get(ROW);
155 get.addColumn(A, A);
156 get.addColumn(B, B);
157 get.addColumn(C, C);
158 table.get(get);
159
160 verifyMethodResult(SimpleRegionObserver.class,
161 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
162 "hadDelete", "hadPrePreparedDeleteTS"},
163 tableName,
164 new Boolean[] {true, true, true, true, false, false}
165 );
166
167 Delete delete = new Delete(ROW);
168 delete.deleteColumn(A, A);
169 delete.deleteColumn(B, B);
170 delete.deleteColumn(C, C);
171 table.delete(delete);
172
173 verifyMethodResult(SimpleRegionObserver.class,
174 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
175 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete", "hadPrePreparedDeleteTS"},
176 tableName,
177 new Boolean[] {true, true, true, true, true, true, true, true}
178 );
179 } finally {
180 util.deleteTable(tableName);
181 table.close();
182 }
183 verifyMethodResult(SimpleRegionObserver.class,
184 new String[] {"getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose"},
185 tableName,
186 new Integer[] {1, 1, 1, 1});
187 }
188
189 @Test (timeout=300000)
190 public void testRowMutation() throws IOException {
191 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRowMutation");
192 Table table = util.createTable(tableName, new byte[][] {A, B, C});
193 try {
194 verifyMethodResult(SimpleRegionObserver.class,
195 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
196 "hadDeleted"},
197 tableName,
198 new Boolean[] {false, false, false, false, false});
199 Put put = new Put(ROW);
200 put.add(A, A, A);
201 put.add(B, B, B);
202 put.add(C, C, C);
203
204 Delete delete = new Delete(ROW);
205 delete.deleteColumn(A, A);
206 delete.deleteColumn(B, B);
207 delete.deleteColumn(C, C);
208
209 RowMutations arm = new RowMutations(ROW);
210 arm.add(put);
211 arm.add(delete);
212 table.mutateRow(arm);
213
214 verifyMethodResult(SimpleRegionObserver.class,
215 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
216 "hadDeleted"},
217 tableName,
218 new Boolean[] {false, false, true, true, true}
219 );
220 } finally {
221 util.deleteTable(tableName);
222 table.close();
223 }
224 }
225
226 @Test (timeout=300000)
227 public void testIncrementHook() throws IOException {
228 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testIncrementHook");
229 Table table = util.createTable(tableName, new byte[][] {A, B, C});
230 try {
231 Increment inc = new Increment(Bytes.toBytes(0));
232 inc.addColumn(A, A, 1);
233
234 verifyMethodResult(SimpleRegionObserver.class,
235 new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"},
236 tableName,
237 new Boolean[] {false, false, false}
238 );
239
240 table.increment(inc);
241
242 verifyMethodResult(SimpleRegionObserver.class,
243 new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"},
244 tableName,
245 new Boolean[] {true, true, true}
246 );
247 } finally {
248 util.deleteTable(tableName);
249 table.close();
250 }
251 }
252
253 @Test (timeout=300000)
254 public void testCheckAndPutHooks() throws IOException {
255 TableName tableName =
256 TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndPutHooks");
257 try (Table table = util.createTable(tableName, new byte[][] {A, B, C})) {
258 Put p = new Put(Bytes.toBytes(0));
259 p.add(A, A, A);
260 table.put(p);
261 p = new Put(Bytes.toBytes(0));
262 p.add(A, A, A);
263 verifyMethodResult(SimpleRegionObserver.class,
264 new String[] {"hadPreCheckAndPut",
265 "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut"},
266 tableName,
267 new Boolean[] {false, false, false}
268 );
269 table.checkAndPut(Bytes.toBytes(0), A, A, A, p);
270 verifyMethodResult(SimpleRegionObserver.class,
271 new String[] {"hadPreCheckAndPut",
272 "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut"},
273 tableName,
274 new Boolean[] {true, true, true}
275 );
276 } finally {
277 util.deleteTable(tableName);
278 }
279 }
280
281 @Test (timeout=300000)
282 public void testCheckAndDeleteHooks() throws IOException {
283 TableName tableName =
284 TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndDeleteHooks");
285 Table table = util.createTable(tableName, new byte[][] {A, B, C});
286 try {
287 Put p = new Put(Bytes.toBytes(0));
288 p.add(A, A, A);
289 table.put(p);
290 Delete d = new Delete(Bytes.toBytes(0));
291 table.delete(d);
292 verifyMethodResult(SimpleRegionObserver.class,
293 new String[] {"hadPreCheckAndDelete",
294 "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"},
295 tableName,
296 new Boolean[] {false, false, false}
297 );
298 table.checkAndDelete(Bytes.toBytes(0), A, A, A, d);
299 verifyMethodResult(SimpleRegionObserver.class,
300 new String[] {"hadPreCheckAndDelete",
301 "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"},
302 tableName,
303 new Boolean[] {true, true, true}
304 );
305 } finally {
306 util.deleteTable(tableName);
307 table.close();
308 }
309 }
310
311 @Test (timeout=300000)
312 public void testAppendHook() throws IOException {
313 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testAppendHook");
314 Table table = util.createTable(tableName, new byte[][] {A, B, C});
315 try {
316 Append app = new Append(Bytes.toBytes(0));
317 app.add(A, A, A);
318
319 verifyMethodResult(SimpleRegionObserver.class,
320 new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"},
321 tableName,
322 new Boolean[] {false, false, false}
323 );
324
325 table.append(app);
326
327 verifyMethodResult(SimpleRegionObserver.class,
328 new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"},
329 tableName,
330 new Boolean[] {true, true, true}
331 );
332 } finally {
333 util.deleteTable(tableName);
334 table.close();
335 }
336 }
337
338 @Test (timeout=300000)
339
340 public void testHBase3583() throws IOException {
341 TableName tableName =
342 TableName.valueOf("testHBase3583");
343 util.createTable(tableName, new byte[][] {A, B, C});
344 util.waitUntilAllRegionsAssigned(tableName);
345
346 verifyMethodResult(SimpleRegionObserver.class,
347 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
348 "wasScannerCloseCalled"},
349 tableName,
350 new Boolean[] {false, false, false, false}
351 );
352
353 Table table = new HTable(util.getConfiguration(), tableName);
354 Put put = new Put(ROW);
355 put.add(A, A, A);
356 table.put(put);
357
358 Get get = new Get(ROW);
359 get.addColumn(A, A);
360 table.get(get);
361
362
363
364 verifyMethodResult(SimpleRegionObserver.class,
365 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
366 "wasScannerCloseCalled"},
367 tableName,
368 new Boolean[] {true, true, false, false}
369 );
370
371 Scan s = new Scan();
372 ResultScanner scanner = table.getScanner(s);
373 try {
374 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
375 }
376 } finally {
377 scanner.close();
378 }
379
380
381 verifyMethodResult(SimpleRegionObserver.class,
382 new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
383 tableName,
384 new Boolean[] {true, true}
385 );
386 util.deleteTable(tableName);
387 table.close();
388 }
389
390 @Test(timeout = 300000)
391 public void testHBASE14489() throws IOException {
392 TableName tableName = TableName.valueOf("testHBASE14489");
393 HTable table = util.createTable(tableName, new byte[][] { A });
394 Put put = new Put(ROW);
395 put.addColumn(A, A, A);
396 table.put(put);
397
398 Scan s = new Scan();
399 s.setFilter(new FilterAllFilter());
400 ResultScanner scanner = table.getScanner(s);
401 try {
402 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
403 }
404 } finally {
405 scanner.close();
406 }
407 verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
408 tableName, new Boolean[] { true });
409 util.deleteTable(tableName);
410 table.close();
411 }
412
413 @Test (timeout=300000)
414
415 public void testHBase3758() throws IOException {
416 TableName tableName =
417 TableName.valueOf("testHBase3758");
418 util.createTable(tableName, new byte[][] {A, B, C});
419
420 verifyMethodResult(SimpleRegionObserver.class,
421 new String[] {"hadDeleted", "wasScannerOpenCalled"},
422 tableName,
423 new Boolean[] {false, false}
424 );
425
426 Table table = new HTable(util.getConfiguration(), tableName);
427 Put put = new Put(ROW);
428 put.add(A, A, A);
429 table.put(put);
430
431 Delete delete = new Delete(ROW);
432 table.delete(delete);
433
434 verifyMethodResult(SimpleRegionObserver.class,
435 new String[] {"hadDeleted", "wasScannerOpenCalled"},
436 tableName,
437 new Boolean[] {true, false}
438 );
439
440 Scan s = new Scan();
441 ResultScanner scanner = table.getScanner(s);
442 try {
443 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
444 }
445 } finally {
446 scanner.close();
447 }
448
449
450 verifyMethodResult(SimpleRegionObserver.class,
451 new String[] {"wasScannerOpenCalled"},
452 tableName,
453 new Boolean[] {true}
454 );
455 util.deleteTable(tableName);
456 table.close();
457 }
458
459
460 public static class EvenOnlyCompactor extends BaseRegionObserver {
461 long lastCompaction;
462 long lastFlush;
463
464 @Override
465 public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
466 Store store, final InternalScanner scanner, final ScanType scanType) {
467 return new InternalScanner() {
468 @Override
469 public boolean next(List<Cell> results) throws IOException {
470 return next(results, NoLimitScannerContext.getInstance());
471 }
472
473 @Override
474 public boolean next(List<Cell> results, ScannerContext scannerContext)
475 throws IOException {
476 List<Cell> internalResults = new ArrayList<Cell>();
477 boolean hasMore;
478 do {
479 hasMore = scanner.next(internalResults, scannerContext);
480 if (!internalResults.isEmpty()) {
481 long row = Bytes.toLong(CellUtil.cloneValue(internalResults.get(0)));
482 if (row % 2 == 0) {
483
484 break;
485 }
486
487 internalResults.clear();
488 }
489 } while (hasMore);
490
491 if (!internalResults.isEmpty()) {
492 results.addAll(internalResults);
493 }
494 return hasMore;
495 }
496
497 @Override
498 public void close() throws IOException {
499 scanner.close();
500 }
501 };
502 }
503
504 @Override
505 public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
506 Store store, StoreFile resultFile) {
507 lastCompaction = EnvironmentEdgeManager.currentTime();
508 }
509
510 @Override
511 public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) {
512 lastFlush = EnvironmentEdgeManager.currentTime();
513 }
514 }
515
516
517
518
519 @Test (timeout=300000)
520 public void testCompactionOverride() throws Exception {
521 TableName compactTable = TableName.valueOf("TestCompactionOverride");
522 Admin admin = util.getHBaseAdmin();
523 if (admin.tableExists(compactTable)) {
524 admin.disableTable(compactTable);
525 admin.deleteTable(compactTable);
526 }
527
528 HTableDescriptor htd = new HTableDescriptor(compactTable);
529 htd.addFamily(new HColumnDescriptor(A));
530 htd.addCoprocessor(EvenOnlyCompactor.class.getName());
531 admin.createTable(htd);
532
533 Table table = new HTable(util.getConfiguration(), compactTable);
534 for (long i=1; i<=10; i++) {
535 byte[] iBytes = Bytes.toBytes(i);
536 Put put = new Put(iBytes);
537 put.setDurability(Durability.SKIP_WAL);
538 put.add(A, A, iBytes);
539 table.put(put);
540 }
541
542 HRegion firstRegion = cluster.getRegions(compactTable).get(0);
543 Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(
544 EvenOnlyCompactor.class.getName());
545 assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
546 EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp;
547
548
549 long ts = System.currentTimeMillis();
550 admin.flush(compactTable);
551
552 for (int i=0; i<10; i++) {
553 if (compactor.lastFlush >= ts) {
554 break;
555 }
556 Thread.sleep(1000);
557 }
558 assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
559 LOG.debug("Flush complete");
560
561 ts = compactor.lastFlush;
562 admin.majorCompact(compactTable);
563
564 for (int i=0; i<30; i++) {
565 if (compactor.lastCompaction >= ts) {
566 break;
567 }
568 Thread.sleep(1000);
569 }
570 LOG.debug("Last compaction was at "+compactor.lastCompaction);
571 assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
572
573
574 ResultScanner scanner = table.getScanner(new Scan());
575 try {
576 for (long i=2; i<=10; i+=2) {
577 Result r = scanner.next();
578 assertNotNull(r);
579 assertFalse(r.isEmpty());
580 byte[] iBytes = Bytes.toBytes(i);
581 assertArrayEquals("Row should be "+i, r.getRow(), iBytes);
582 assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes);
583 }
584 } finally {
585 scanner.close();
586 }
587 table.close();
588 }
589
590 @Test (timeout=300000)
591 public void bulkLoadHFileTest() throws Exception {
592 String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest";
593 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".bulkLoadHFileTest");
594 Configuration conf = util.getConfiguration();
595 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
596 try {
597 verifyMethodResult(SimpleRegionObserver.class,
598 new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
599 tableName,
600 new Boolean[] {false, false}
601 );
602
603 FileSystem fs = util.getTestFileSystem();
604 final Path dir = util.getDataTestDirOnTestFS(testName).makeQualified(fs);
605 Path familyDir = new Path(dir, Bytes.toString(A));
606
607 createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);
608
609
610 new LoadIncrementalHFiles(conf).doBulkLoad(dir, table);
611
612 verifyMethodResult(SimpleRegionObserver.class,
613 new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
614 tableName,
615 new Boolean[] {true, true}
616 );
617 } finally {
618 util.deleteTable(tableName);
619 table.close();
620 }
621 }
622
623 @Ignore
624 @Test (timeout=300000)
625 public void testRecovery() throws Exception {
626 LOG.info(TestRegionObserverInterface.class.getName() +".testRecovery");
627 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
628 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
629 try {
630 JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
631 ServerName sn2 = rs1.getRegionServer().getServerName();
632 String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
633
634 util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
635 while (!sn2.equals(table.getRegionLocations().firstEntry().getValue() )){
636 Thread.sleep(100);
637 }
638
639 Put put = new Put(ROW);
640 put.add(A, A, A);
641 put.add(B, B, B);
642 put.add(C, C, C);
643 table.put(put);
644
645 verifyMethodResult(SimpleRegionObserver.class,
646 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
647 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
648 tableName,
649 new Boolean[] {false, false, true, true, true, true, false}
650 );
651
652 verifyMethodResult(SimpleRegionObserver.class,
653 new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
654 "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
655 tableName,
656 new Integer[] {0, 0, 1, 1, 0, 0});
657
658 cluster.killRegionServer(rs1.getRegionServer().getServerName());
659 Threads.sleep(1000);
660 util.waitUntilAllRegionsAssigned(tableName);
661 LOG.info("All regions assigned");
662
663 verifyMethodResult(SimpleRegionObserver.class,
664 new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
665 "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
666 tableName,
667 new Integer[]{1, 1, 0, 0, 0, 0});
668 } finally {
669 util.deleteTable(tableName);
670 table.close();
671 }
672 }
673
674 @Ignore
675 @Test (timeout=300000)
676 public void testLegacyRecovery() throws Exception {
677 LOG.info(TestRegionObserverInterface.class.getName() +".testLegacyRecovery");
678 TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testLegacyRecovery");
679 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
680 try {
681 JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
682 ServerName sn2 = rs1.getRegionServer().getServerName();
683 String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
684
685 util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
686 while (!sn2.equals(table.getRegionLocations().firstEntry().getValue() )){
687 Thread.sleep(100);
688 }
689
690 Put put = new Put(ROW);
691 put.add(A, A, A);
692 put.add(B, B, B);
693 put.add(C, C, C);
694 table.put(put);
695
696 verifyMethodResult(SimpleRegionObserver.Legacy.class,
697 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
698 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
699 tableName,
700 new Boolean[] {false, false, true, true, true, true, false}
701 );
702
703 verifyMethodResult(SimpleRegionObserver.Legacy.class,
704 new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
705 "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
706 tableName,
707 new Integer[] {0, 0, 1, 1, 0, 0});
708
709 cluster.killRegionServer(rs1.getRegionServer().getServerName());
710 Threads.sleep(1000);
711 util.waitUntilAllRegionsAssigned(tableName);
712 LOG.info("All regions assigned");
713
714 verifyMethodResult(SimpleRegionObserver.Legacy.class,
715 new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
716 "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
717 tableName,
718 new Integer[]{1, 1, 0, 0, 1, 1});
719 } finally {
720 util.deleteTable(tableName);
721 table.close();
722 }
723 }
724
725 @Test (timeout=300000)
726 public void testPreWALRestoreSkip() throws Exception {
727 LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
728 TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
729 HTable table = util.createTable(tableName, new byte[][] { A, B, C });
730
731 JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
732 ServerName sn2 = rs1.getRegionServer().getServerName();
733 String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
734
735 util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
736 while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
737 Thread.sleep(100);
738 }
739
740 Put put = new Put(ROW);
741 put.add(A, A, A);
742 put.add(B, B, B);
743 put.add(C, C, C);
744 table.put(put);
745 table.flushCommits();
746
747 cluster.killRegionServer(rs1.getRegionServer().getServerName());
748 Threads.sleep(20000);
749 util.waitUntilAllRegionsAssigned(tableName);
750
751 verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore",
752 "getCtPostWALRestore", "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
753 tableName,
754 new Integer[] {0, 0, 0, 0});
755
756 util.deleteTable(tableName);
757 table.close();
758 }
759
760
761 private void testPreWALAppendHook(Table table, TableName tableName) throws IOException {
762 int expectedCalls = 0;
763 String [] methodArray = new String[1];
764 methodArray[0] = "getCtPreWALAppend";
765 Object[] resultArray = new Object[1];
766
767 Put p = new Put(ROW);
768 p.addColumn(A, A, A);
769 table.put(p);
770 resultArray[0] = ++expectedCalls;
771 verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
772
773 Append a = new Append(ROW);
774 a.add(B, B, B);
775 table.append(a);
776 resultArray[0] = ++expectedCalls;
777 verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
778
779 Increment i = new Increment(ROW);
780 i.addColumn(C, C, 1);
781 table.increment(i);
782 resultArray[0] = ++expectedCalls;
783 verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
784
785 Delete d = new Delete(ROW);
786 table.delete(d);
787 resultArray[0] = ++expectedCalls;
788 verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
789 }
790
791 @Test
792 public void testPreWALAppend() throws Exception {
793 SimpleRegionObserver sro = new SimpleRegionObserver();
794 ObserverContext ctx = Mockito.mock(ObserverContext.class);
795 WALKey key = new WALKey(Bytes.toBytes("region"), TEST_TABLE,
796 EnvironmentEdgeManager.currentTime());
797 WALEdit edit = new WALEdit();
798 sro.preWALAppend(ctx, key, edit);
799 Assert.assertEquals(1, key.getExtendedAttributes().size());
800 Assert.assertArrayEquals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
801 key.getExtendedAttribute(Integer.toString(sro.getCtPreWALAppend())));
802 }
803
804 @Test
805 public void testPreWALAppendIsWrittenToWAL() throws Exception {
806 final TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() +
807 ".testPreWALAppendIsWrittenToWAL");
808 HTableDescriptor htd = new HTableDescriptor(tableName);
809 htd.addFamily(new HColumnDescriptor(A));
810 htd.addFamily(new HColumnDescriptor(B));
811 htd.addFamily(new HColumnDescriptor(C));
812 htd.addCoprocessor(SimpleRegionObserver.class.getName());
813 Table table = util.createTable(htd, null);
814 PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
815 List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
816
817 HRegion region = regions.get(0);
818 region.getWAL().registerWALActionsListener(listener);
819 testPreWALAppendHook(table, tableName);
820 boolean[] expectedResults = {true, true, true, true};
821 Assert.assertArrayEquals(expectedResults, listener.getWalKeysCorrectArray());
822
823 }
824
825 @Test
826 public void testPreWALAppendNotCalledOnMetaEdit() throws Exception {
827 final TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() +
828 ".testPreWALAppendNotCalledOnMetaEdt");
829
830 HTableDescriptor td = new HTableDescriptor(tableName);
831 td.addCoprocessor(SimpleRegionObserver.class.getName());
832 td.addFamily(new HColumnDescriptor(FAMILY));
833 Table table = util.createTable(td, new byte[][] { A, B, C });
834
835 PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
836 List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
837
838 HRegion region = regions.get(0);
839
840 region.getWAL().registerWALActionsListener(listener);
841
842 region.flush(true);
843
844 region.compact(false);
845
846 region.close();
847
848
849 String[] methods = new String[] {"getCtPreWALAppend"};
850 Object[] expectedResult = new Integer[]{0};
851 verifyMethodResult(SimpleRegionObserver.class, methods, tableName, expectedResult);
852 }
853
854
855 private void verifyMethodResult(Class<?> c, String methodName[], TableName tableName,
856 Object value[]) throws IOException {
857 try {
858 for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
859 if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()){
860 continue;
861 }
862 for (HRegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer().getRSRpcServices())) {
863 if (!r.getTable().equals(tableName)) {
864 continue;
865 }
866 RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).
867 getCoprocessorHost();
868
869 Coprocessor cp = cph.findCoprocessor(c.getName());
870 assertNotNull(cp);
871 for (int i = 0; i < methodName.length; ++i) {
872 Method m = c.getMethod(methodName[i]);
873 Object o = m.invoke(cp);
874 assertTrue("Result of " + c.getName() + "." + methodName[i]
875 + " is expected to be " + value[i].toString()
876 + ", while we get " + o.toString(), o.equals(value[i]));
877 }
878 }
879 }
880 } catch (Exception e) {
881 throw new IOException(e.toString());
882 }
883 }
884
885 private static void createHFile(
886 Configuration conf,
887 FileSystem fs, Path path,
888 byte[] family, byte[] qualifier) throws IOException {
889 HFileContext context = new HFileContextBuilder().build();
890 HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
891 .withPath(fs, path)
892 .withFileContext(context)
893 .create();
894 long now = System.currentTimeMillis();
895 try {
896 for (int i =1;i<=9;i++) {
897 KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+""));
898 writer.append(kv);
899 }
900 } finally {
901 writer.close();
902 }
903 }
904
905 private static class PreWALAppendWALActionsListener implements WALActionsListener {
906 boolean[] walKeysCorrect = {false, false, false, false};
907
908 @Override
909 public void postAppend(long entryLen, long elapsedTimeMillis,
910 WALKey logKey, WALEdit logEdit) throws IOException {
911 for (int k = 0; k < 4; k++) {
912 if (!walKeysCorrect[k]) {
913 walKeysCorrect[k] = Arrays.equals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
914 logKey.getExtendedAttribute(Integer.toString(k + 1)));
915 }
916 }
917 }
918
919 @Override
920 public void postSync(long timeInNanos, int handlerSyncs) {
921
922 }
923
924 boolean[] getWalKeysCorrectArray() {
925 return walKeysCorrect;
926 }
927 @Override
928 public void preLogRoll(Path oldPath, Path newPath) throws IOException {
929
930 }
931
932 @Override
933 public void postLogRoll(Path oldPath, Path newPath) throws IOException {
934
935 }
936
937 @Override
938 public void preLogArchive(Path oldPath, Path newPath) throws IOException {
939
940 }
941
942 @Override
943 public void postLogArchive(Path oldPath, Path newPath) throws IOException {
944
945 }
946
947 @Override
948 public void logRollRequested(RollRequestReason reason) {
949
950 }
951
952 @Override
953 public void logCloseRequested() {
954
955 }
956
957 @Override
958 public void visitLogEntryBeforeWrite(HRegionInfo info, WALKey logKey,
959 WALEdit logEdit) {
960
961 }
962
963 @Override
964 public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey,
965 WALEdit logEdit) throws IOException {
966
967 }
968 }
969 }