1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.client;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.RegionLocations;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.Waiter;
38 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
39 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
40 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
41 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
42 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
43 import org.apache.hadoop.hbase.protobuf.RequestConverter;
44 import org.apache.hadoop.hbase.regionserver.RegionScanner;
45 import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
46 import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
47 import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
48 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.Pair;
51 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
52 import org.junit.AfterClass;
53 import org.junit.Assert;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56 import org.junit.experimental.categories.Category;
57
58 import java.io.IOException;
59 import java.util.ArrayList;
60 import java.util.Arrays;
61 import java.util.List;
62 import java.util.concurrent.CountDownLatch;
63 import java.util.concurrent.TimeUnit;
64 import java.util.concurrent.atomic.AtomicLong;
65 import java.util.concurrent.atomic.AtomicReference;
66
67 @Category(MediumTests.class)
68 public class TestReplicaWithCluster {
69 private static final Log LOG = LogFactory.getLog(TestReplicaWithCluster.class);
70
71 private static final int NB_SERVERS = 3;
72 private static final byte[] row = TestReplicaWithCluster.class.getName().getBytes();
73 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
74
75
76 private static HBaseTestingUtility HTU2;
77 private static final byte[] f = HConstants.CATALOG_FAMILY;
78
79 private final static int REFRESH_PERIOD = 1000;
80 private final static int META_SCAN_TIMEOUT_IN_MILLISEC = 200;
81
82
83
84
85 public static class SlowMeCopro extends BaseRegionObserver {
86 static final AtomicLong sleepTime = new AtomicLong(0);
87 static final AtomicReference<CountDownLatch> cdl =
88 new AtomicReference<CountDownLatch>(new CountDownLatch(0));
89
90 public SlowMeCopro() {
91 }
92
93 @Override
94 public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
95 final Get get, final List<Cell> results) throws IOException {
96
97 if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() == 0) {
98 CountDownLatch latch = cdl.get();
99 try {
100 if (sleepTime.get() > 0) {
101 LOG.info("Sleeping for " + sleepTime.get() + " ms");
102 Thread.sleep(sleepTime.get());
103 } else if (latch.getCount() > 0) {
104 LOG.info("Waiting for the counterCountDownLatch");
105 latch.await(2, TimeUnit.MINUTES);
106 if (latch.getCount() > 0) {
107 throw new RuntimeException("Can't wait more");
108 }
109 }
110 } catch (InterruptedException e1) {
111 LOG.error(e1);
112 }
113 } else {
114 LOG.info("We're not the primary replicas.");
115 }
116 }
117 }
118
119
120
121
122 public static class RegionServerStoppedCopro extends BaseRegionObserver {
123
124 public RegionServerStoppedCopro() {
125 }
126
127 @Override
128 public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
129 final Get get, final List<Cell> results) throws IOException {
130
131 int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId();
132
133
134 if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() <= 1) {
135 LOG.info("Throw Region Server Stopped Exceptoin for replica id " + replicaId);
136 throw new RegionServerStoppedException("Server " +
137 e.getEnvironment().getRegionServerServices().getServerName()
138 + " not running");
139 } else {
140 LOG.info("We're replica region " + replicaId);
141 }
142 }
143
144 @Override
145 public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e,
146 final Scan scan, final RegionScanner s) throws IOException {
147
148 int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId();
149
150
151 if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() <= 1) {
152 LOG.info("Throw Region Server Stopped Exceptoin for replica id " + replicaId);
153 throw new RegionServerStoppedException("Server " +
154 e.getEnvironment().getRegionServerServices().getServerName()
155 + " not running");
156 } else {
157 LOG.info("We're replica region " + replicaId);
158 }
159
160 return null;
161 }
162
163 @Override
164 public void postGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> c,
165 final byte [] row, final byte [] family, final Result result)
166 throws IOException {
167
168 }
169 }
170
171
172
173
174 public static class RegionServerHostingPrimayMetaRegionSlowOrStopCopro extends BaseRegionObserver {
175 static boolean slowDownPrimaryMetaScan = false;
176 static boolean throwException = false;
177
178 @Override
179 public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
180 final Get get, final List<Cell> results) throws IOException {
181
182 int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId();
183
184
185 if (throwException) {
186 if (!e.getEnvironment().getRegion().getRegionInfo().isMetaRegion() && (replicaId == 0)) {
187 LOG.info("Get, throw Region Server Stopped Exceptoin for region " + e.getEnvironment()
188 .getRegion().getRegionInfo());
189 throw new RegionServerStoppedException("Server " +
190 e.getEnvironment().getRegionServerServices().getServerName() + " not running");
191 }
192 } else {
193 LOG.info("Get, We're replica region " + replicaId);
194 }
195 }
196
197 @Override
198 public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e,
199 final Scan scan, final RegionScanner s) throws IOException {
200
201 int replicaId = e.getEnvironment().getRegion().getRegionInfo().getReplicaId();
202
203
204 if (e.getEnvironment().getRegion().getRegionInfo().isMetaRegion() && (replicaId == 0)) {
205 if (slowDownPrimaryMetaScan) {
206 LOG.info("Scan with primary meta region, slow down a bit");
207 try {
208 Thread.sleep(META_SCAN_TIMEOUT_IN_MILLISEC - 50);
209 } catch (InterruptedException ie) {
210
211 }
212 }
213
214
215 if (throwException) {
216 LOG.info("Scan, throw Region Server Stopped Exceptoin for replica " + e.getEnvironment()
217 .getRegion().getRegionInfo());
218
219 throw new RegionServerStoppedException("Server " +
220 e.getEnvironment().getRegionServerServices().getServerName() + " not running");
221 } else {
222 LOG.info("Scan, We're replica region " + replicaId);
223 }
224 } else {
225 LOG.info("Scan, We're replica region " + replicaId);
226 }
227
228 return null;
229 }
230 }
231
232 @BeforeClass
233 public static void beforeClass() throws Exception {
234
235 HTU.getConfiguration().setInt(
236 StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);
237
238 HTU.getConfiguration().setFloat("hbase.regionserver.logroll.multiplier", 0.0001f);
239 HTU.getConfiguration().setInt("replication.source.size.capacity", 10240);
240 HTU.getConfiguration().setLong("replication.source.sleepforretries", 100);
241 HTU.getConfiguration().setInt("hbase.regionserver.maxlogs", 2);
242 HTU.getConfiguration().setLong("hbase.master.logcleaner.ttl", 10);
243 HTU.getConfiguration().setInt("zookeeper.recovery.retry", 1);
244 HTU.getConfiguration().setInt("zookeeper.recovery.retry.intervalmill", 10);
245
246
247 HTU.getConfiguration().setInt("hbase.client.primaryCallTimeout.get", 1000000);
248 HTU.getConfiguration().setInt("hbase.client.primaryCallTimeout.scan", 1000000);
249
250
251 HTU.getConfiguration().setInt("hbase.meta.replica.count", 2);
252
253
254 HTU.getConfiguration().set("hbase.balancer.tablesOnMaster", "none");
255
256
257 HTU.getConfiguration().set("hbase.coprocessor.region.classes",
258 RegionServerHostingPrimayMetaRegionSlowOrStopCopro.class.getName());
259
260 HTU.getConfiguration().setInt(HConstants.HBASE_CLIENT_META_REPLICA_SCAN_TIMEOUT,
261 META_SCAN_TIMEOUT_IN_MILLISEC * 1000);
262
263 HTU.startMiniCluster(NB_SERVERS);
264 HTU.getHBaseCluster().startMaster();
265 }
266
267 @AfterClass
268 public static void afterClass() throws Exception {
269 if (HTU2 != null) {
270 HTU2.shutdownMiniCluster();
271 }
272 HTU.shutdownMiniCluster();
273 }
274
275 @Test (timeout=30000)
276 public void testCreateDeleteTable() throws IOException {
277
278 HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
279 hdt.setRegionReplication(NB_SERVERS);
280 hdt.addCoprocessor(SlowMeCopro.class.getName());
281 Table table = HTU.createTable(hdt, new byte[][]{f}, HTU.getConfiguration());
282
283 Put p = new Put(row);
284 p.add(f, row, row);
285 table.put(p);
286
287 Get g = new Get(row);
288 Result r = table.get(g);
289 Assert.assertFalse(r.isStale());
290
291 try {
292
293 SlowMeCopro.cdl.set(new CountDownLatch(1));
294 g = new Get(row);
295 g.setConsistency(Consistency.TIMELINE);
296 r = table.get(g);
297 Assert.assertTrue(r.isStale());
298 SlowMeCopro.cdl.get().countDown();
299 } finally {
300 SlowMeCopro.cdl.get().countDown();
301 SlowMeCopro.sleepTime.set(0);
302 }
303
304 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
305 HTU.deleteTable(hdt.getTableName());
306 }
307
308 @Test (timeout=120000)
309 public void testChangeTable() throws Exception {
310 HTableDescriptor hdt = HTU.createTableDescriptor("testChangeTable");
311 hdt.setRegionReplication(NB_SERVERS);
312 hdt.addCoprocessor(SlowMeCopro.class.getName());
313 Table table = HTU.createTable(hdt, new byte[][]{f}, HTU.getConfiguration());
314
315
316 Put p = new Put(row);
317 p.add(f, row, row);
318 table.put(p);
319
320 Get g = new Get(row);
321 Result r = table.get(g);
322 Assert.assertFalse(r.isStale());
323
324
325 HTableDescriptor bHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
326 HColumnDescriptor hcd = new HColumnDescriptor(row);
327 hdt.addFamily(hcd);
328 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
329 HTU.getHBaseAdmin().modifyTable(hdt.getTableName(), hdt);
330 HTU.getHBaseAdmin().enableTable(hdt.getTableName());
331 HTableDescriptor nHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
332 Assert.assertEquals("fams=" + Arrays.toString(nHdt.getColumnFamilies()),
333 bHdt.getColumnFamilies().length + 1, nHdt.getColumnFamilies().length);
334
335 p = new Put(row);
336 p.add(row, row, row);
337 table.put(p);
338
339 g = new Get(row);
340 r = table.get(g);
341 Assert.assertFalse(r.isStale());
342
343 try {
344 SlowMeCopro.cdl.set(new CountDownLatch(1));
345 g = new Get(row);
346 g.setConsistency(Consistency.TIMELINE);
347 r = table.get(g);
348 Assert.assertTrue(r.isStale());
349 } finally {
350 SlowMeCopro.cdl.get().countDown();
351 SlowMeCopro.sleepTime.set(0);
352 }
353
354 HTU.getHBaseCluster().stopMaster(0);
355 Admin admin = new HBaseAdmin(HTU.getConfiguration());
356 nHdt =admin.getTableDescriptor(hdt.getTableName());
357 Assert.assertEquals("fams=" + Arrays.toString(nHdt.getColumnFamilies()),
358 bHdt.getColumnFamilies().length + 1, nHdt.getColumnFamilies().length);
359
360 admin.disableTable(hdt.getTableName());
361 admin.deleteTable(hdt.getTableName());
362 HTU.getHBaseCluster().startMaster();
363 admin.close();
364 }
365
366 @SuppressWarnings("deprecation")
367 @Test (timeout=300000)
368 public void testReplicaAndReplication() throws Exception {
369 HTableDescriptor hdt = HTU.createTableDescriptor("testReplicaAndReplication");
370 hdt.setRegionReplication(NB_SERVERS);
371
372 HColumnDescriptor fam = new HColumnDescriptor(row);
373 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
374 hdt.addFamily(fam);
375
376 hdt.addCoprocessor(SlowMeCopro.class.getName());
377 HTU.getHBaseAdmin().createTable(hdt, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
378
379 Configuration conf2 = HBaseConfiguration.create(HTU.getConfiguration());
380 conf2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
381 conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
382 MiniZooKeeperCluster miniZK = HTU.getZkCluster();
383
384 HTU2 = new HBaseTestingUtility(conf2);
385 HTU2.setZkCluster(miniZK);
386 HTU2.startMiniCluster(NB_SERVERS);
387 LOG.info("Setup second Zk");
388 HTU2.getHBaseAdmin().createTable(hdt, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
389
390 ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
391 ReplicationPeerConfig rpc = new ReplicationPeerConfig();
392 rpc.setClusterKey(HTU2.getClusterKey());
393 admin.addPeer("2", rpc);
394 admin.close();
395
396 Put p = new Put(row);
397 p.add(row, row, row);
398 final Table table = new HTable(HTU.getConfiguration(), hdt.getTableName());
399 table.put(p);
400
401 HTU.getHBaseAdmin().flush(table.getName());
402 LOG.info("Put & flush done on the first cluster. Now doing a get on the same cluster.");
403
404 Waiter.waitFor(HTU.getConfiguration(), 1000, new Waiter.Predicate<Exception>() {
405 @Override
406 public boolean evaluate() throws Exception {
407 try {
408 SlowMeCopro.cdl.set(new CountDownLatch(1));
409 Get g = new Get(row);
410 g.setConsistency(Consistency.TIMELINE);
411 Result r = table.get(g);
412 Assert.assertTrue(r.isStale());
413 return !r.isEmpty();
414 } finally {
415 SlowMeCopro.cdl.get().countDown();
416 SlowMeCopro.sleepTime.set(0);
417 }
418 }});
419 table.close();
420 LOG.info("stale get on the first cluster done. Now for the second.");
421
422 final Table table2 = new HTable(HTU.getConfiguration(), hdt.getTableName());
423 Waiter.waitFor(HTU.getConfiguration(), 1000, new Waiter.Predicate<Exception>() {
424 @Override
425 public boolean evaluate() throws Exception {
426 try {
427 SlowMeCopro.cdl.set(new CountDownLatch(1));
428 Get g = new Get(row);
429 g.setConsistency(Consistency.TIMELINE);
430 Result r = table2.get(g);
431 Assert.assertTrue(r.isStale());
432 return !r.isEmpty();
433 } finally {
434 SlowMeCopro.cdl.get().countDown();
435 SlowMeCopro.sleepTime.set(0);
436 }
437 }});
438 table2.close();
439
440 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
441 HTU.deleteTable(hdt.getTableName());
442
443 HTU2.getHBaseAdmin().disableTable(hdt.getTableName());
444 HTU2.deleteTable(hdt.getTableName());
445
446
447
448 }
449
450 @Test (timeout=30000)
451 public void testBulkLoad() throws IOException {
452
453 LOG.debug("Creating test table");
454 HTableDescriptor hdt = HTU.createTableDescriptor("testBulkLoad");
455 hdt.setRegionReplication(NB_SERVERS);
456 hdt.addCoprocessor(SlowMeCopro.class.getName());
457 Table table = HTU.createTable(hdt, new byte[][]{f}, HTU.getConfiguration());
458
459
460 LOG.debug("Creating test data");
461 Path dir = HTU.getDataTestDirOnTestFS("testBulkLoad");
462 final int numRows = 10;
463 final byte[] qual = Bytes.toBytes("qual");
464 final byte[] val = Bytes.toBytes("val");
465 final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>();
466 for (HColumnDescriptor col : hdt.getColumnFamilies()) {
467 Path hfile = new Path(dir, col.getNameAsString());
468 TestHRegionServerBulkLoad.createHFile(HTU.getTestFileSystem(), hfile, col.getName(),
469 qual, val, numRows);
470 famPaths.add(new Pair<byte[], String>(col.getName(), hfile.toString()));
471 }
472
473
474 LOG.debug("Loading test data");
475 @SuppressWarnings("deprecation")
476 final HConnection conn = HTU.getHBaseAdmin().getConnection();
477 RegionServerCallable<Void> callable = new RegionServerCallable<Void>(
478 conn, hdt.getTableName(), TestHRegionServerBulkLoad.rowkey(0)) {
479 @Override
480 public Void call(int timeout) throws Exception {
481 LOG.debug("Going to connect to server " + getLocation() + " for row "
482 + Bytes.toStringBinary(getRow()));
483 byte[] regionName = getLocation().getRegionInfo().getRegionName();
484 BulkLoadHFileRequest request =
485 RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true);
486 getStub().bulkLoadHFile(null, request);
487 return null;
488 }
489 };
490 RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(HTU.getConfiguration());
491 RpcRetryingCaller<Void> caller = factory.<Void> newCaller();
492 caller.callWithRetries(callable, 10000);
493
494
495 LOG.debug("Verifying data load");
496 for (int i = 0; i < numRows; i++) {
497 byte[] row = TestHRegionServerBulkLoad.rowkey(i);
498 Get g = new Get(row);
499 Result r = table.get(g);
500 Assert.assertFalse(r.isStale());
501 }
502
503
504 LOG.debug("Verifying replica queries");
505 try {
506 SlowMeCopro.cdl.set(new CountDownLatch(1));
507 for (int i = 0; i < numRows; i++) {
508 byte[] row = TestHRegionServerBulkLoad.rowkey(i);
509 Get g = new Get(row);
510 g.setConsistency(Consistency.TIMELINE);
511 Result r = table.get(g);
512 Assert.assertTrue(r.isStale());
513 }
514 SlowMeCopro.cdl.get().countDown();
515 } finally {
516 SlowMeCopro.cdl.get().countDown();
517 SlowMeCopro.sleepTime.set(0);
518 }
519
520 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
521 HTU.deleteTable(hdt.getTableName());
522 }
523
524 @Test
525 public void testReplicaGetWithPrimaryDown() throws IOException {
526
527 HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
528 hdt.setRegionReplication(NB_SERVERS);
529 hdt.addCoprocessor(RegionServerStoppedCopro.class.getName());
530 try {
531
532 HTU.getConfiguration().setInt("hbase.client.retries.number", 1);
533
534 Table table = HTU.createTable(hdt, new byte[][] { f }, null);
535
536 Put p = new Put(row);
537 p.addColumn(f, row, row);
538 table.put(p);
539
540
541 HTU.flush(table.getName());
542
543
544 try {
545 Thread.sleep(2 * REFRESH_PERIOD);
546 } catch (InterruptedException e1) {
547 LOG.error(e1);
548 }
549
550
551 Get g = new Get(row);
552 g.setConsistency(Consistency.TIMELINE);
553 Result r = table.get(g);
554 Assert.assertTrue(r.isStale());
555 } finally {
556 HTU.getConfiguration().unset("hbase.client.retries.number");
557 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
558 HTU.deleteTable(hdt.getTableName());
559 }
560 }
561
562 @Test
563 public void testReplicaScanWithPrimaryDown() throws IOException {
564
565 HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
566 hdt.setRegionReplication(NB_SERVERS);
567 hdt.addCoprocessor(RegionServerStoppedCopro.class.getName());
568
569 try {
570
571 HTU.getConfiguration().setInt("hbase.client.retries.number", 1);
572
573 Table table = HTU.createTable(hdt, new byte[][] { f }, null);
574
575 Put p = new Put(row);
576 p.addColumn(f, row, row);
577 table.put(p);
578
579
580 HTU.flush(table.getName());
581
582
583 try {
584 Thread.sleep(2 * REFRESH_PERIOD);
585 } catch (InterruptedException e1) {
586 LOG.error(e1);
587 }
588
589
590
591 Scan scan = new Scan();
592
593
594 scan.addFamily(f);
595 scan.setConsistency(Consistency.TIMELINE);
596
597
598 ResultScanner scanner = table.getScanner(scan);
599
600 Result r = scanner.next();
601
602 Assert.assertTrue(r.isStale());
603 } finally {
604 HTU.getConfiguration().unset("hbase.client.retries.number");
605 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
606 HTU.deleteTable(hdt.getTableName());
607 }
608 }
609
610 @Test
611 public void testReplicaGetWithRpcClientImpl() throws IOException {
612 HTU.getConfiguration().setBoolean("hbase.ipc.client.specificThreadForWriting", true);
613 HTU.getConfiguration().set("hbase.rpc.client.impl", "org.apache.hadoop.hbase.ipc.RpcClientImpl");
614
615 HTableDescriptor hdt = HTU.createTableDescriptor("testReplicaGetWithRpcClientImpl");
616 hdt.setRegionReplication(NB_SERVERS);
617 hdt.addCoprocessor(SlowMeCopro.class.getName());
618
619 try {
620 Table table = HTU.createTable(hdt, new byte[][] { f }, null);
621
622 Put p = new Put(row);
623 p.addColumn(f, row, row);
624 table.put(p);
625
626
627 HTU.flush(table.getName());
628
629
630 try {
631 Thread.sleep(2 * REFRESH_PERIOD);
632 } catch (InterruptedException e1) {
633 LOG.error(e1);
634 }
635
636 try {
637
638 Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
639 Table t = connection.getTable(hdt.getTableName());
640
641
642 SlowMeCopro.cdl.set(new CountDownLatch(1));
643 Get g = new Get(row);
644 g.setConsistency(Consistency.TIMELINE);
645 Result r = t.get(g);
646 Assert.assertTrue(r.isStale());
647 SlowMeCopro.cdl.get().countDown();
648 } finally {
649 SlowMeCopro.cdl.get().countDown();
650 SlowMeCopro.sleepTime.set(0);
651 }
652 } finally {
653 HTU.getConfiguration().unset("hbase.ipc.client.specificThreadForWriting");
654 HTU.getConfiguration().unset("hbase.rpc.client.impl");
655 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
656 HTU.deleteTable(hdt.getTableName());
657 }
658 }
659
660
661
662
663 @Test
664 public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException {
665 HTU.getHBaseAdmin().setBalancerRunning(false, true);
666
667 ((ConnectionManager.HConnectionImplementation) HTU.getHBaseAdmin().getConnection()).
668 setUseMetaReplicas(true);
669
670
671 HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion");
672 hdt.setRegionReplication(2);
673 try {
674
675 HTU.createTable(hdt, new byte[][] { f }, null);
676
677 RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = true;
678
679
680 RegionLocations url = ((ClusterConnection) HTU.getConnection())
681 .locateRegion(hdt.getTableName(), row, false, false);
682
683 } finally {
684 RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = false;
685 ((ConnectionManager.HConnectionImplementation) HTU.getHBaseAdmin().getConnection()).
686 setUseMetaReplicas(false);
687 HTU.getHBaseAdmin().setBalancerRunning(true, true);
688 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
689 HTU.deleteTable(hdt.getTableName());
690 }
691 }
692
693
694
695
696
697
698 @Test
699 public void testReplicaGetWithPrimaryAndMetaDown() throws IOException, InterruptedException {
700 HTU.getHBaseAdmin().setBalancerRunning(false, true);
701
702 ((ConnectionManager.HConnectionImplementation)HTU.getHBaseAdmin().getConnection()).
703 setUseMetaReplicas(true);
704
705
706 HTableDescriptor hdt = HTU.createTableDescriptor("testReplicaGetWithPrimaryAndMetaDown");
707 hdt.setRegionReplication(2);
708 try {
709
710 Table table = HTU.createTable(hdt, new byte[][] { f }, null);
711
712
713 RegionLocations mrl = ((ClusterConnection) HTU.getConnection())
714 .locateRegion(TableName.META_TABLE_NAME,
715 HConstants.EMPTY_START_ROW, false, false);
716
717
718 RegionLocations url = ((ClusterConnection) HTU.getConnection())
719 .locateRegion(hdt.getTableName(), row, false, false);
720
721
722 if (!url.getDefaultRegionLocation().getServerName().equals(
723 mrl.getDefaultRegionLocation().getServerName())) {
724 HTU.moveRegionAndWait(url.getDefaultRegionLocation().getRegionInfo(),
725 mrl.getDefaultRegionLocation().getServerName());
726 }
727
728
729
730 if (url.getRegionLocation(1).getServerName().equals(mrl.getDefaultRegionLocation()
731 .getServerName())) {
732 HTU.moveRegionAndWait(url.getRegionLocation(1).getRegionInfo(),
733 url.getDefaultRegionLocation().getServerName());
734 }
735
736
737 while (true) {
738 mrl = ((ClusterConnection) HTU.getConnection())
739 .locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false);
740
741
742 url = ((ClusterConnection) HTU.getConnection())
743 .locateRegion(hdt.getTableName(), row, false, true);
744
745 LOG.info("meta locations " + mrl);
746 LOG.info("table locations " + url);
747 ServerName a = url.getDefaultRegionLocation().getServerName();
748 ServerName b = mrl.getDefaultRegionLocation().getServerName();
749 if(a.equals(b)) {
750 break;
751 } else {
752 LOG.info("Waiting for new region info to be updated in meta table");
753 Thread.sleep(100);
754 }
755 }
756
757 Put p = new Put(row);
758 p.addColumn(f, row, row);
759 table.put(p);
760
761
762 HTU.flush(table.getName());
763
764
765 try {
766 Thread.sleep(2 * REFRESH_PERIOD);
767 } catch (InterruptedException e1) {
768 LOG.error(e1);
769 }
770
771
772 RegionServerHostingPrimayMetaRegionSlowOrStopCopro.throwException = true;
773
774
775 Get g = new Get(row);
776 g.setConsistency(Consistency.TIMELINE);
777 Result r = table.get(g);
778 Assert.assertTrue(r.isStale());
779
780
781 r = table.get(g);
782 Assert.assertTrue(r.isStale());
783
784 } finally {
785 ((ConnectionManager.HConnectionImplementation)HTU.getHBaseAdmin().getConnection()).
786 setUseMetaReplicas(false);
787 RegionServerHostingPrimayMetaRegionSlowOrStopCopro.throwException = false;
788 HTU.getHBaseAdmin().setBalancerRunning(true, true);
789 HTU.getHBaseAdmin().disableTable(hdt.getTableName());
790 HTU.deleteTable(hdt.getTableName());
791 }
792 }
793 }