1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication;
21
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertNotNull;
26 import static org.junit.Assert.assertNull;
27 import static org.junit.Assert.assertTrue;
28 import static org.junit.Assert.fail;
29
30 import java.io.IOException;
31 import java.util.ArrayList;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Map;
35
36 import org.apache.commons.logging.Log;
37 import org.apache.commons.logging.LogFactory;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.hbase.HBaseConfiguration;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
45 import org.apache.hadoop.hbase.testclassification.LargeTests;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.client.Admin;
48 import org.apache.hadoop.hbase.client.Connection;
49 import org.apache.hadoop.hbase.client.ConnectionFactory;
50 import org.apache.hadoop.hbase.client.Delete;
51 import org.apache.hadoop.hbase.client.Get;
52 import org.apache.hadoop.hbase.client.Put;
53 import org.apache.hadoop.hbase.client.Result;
54 import org.apache.hadoop.hbase.client.Table;
55 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
56 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
59 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
60 import org.junit.AfterClass;
61 import org.junit.BeforeClass;
62 import org.junit.Test;
63 import org.junit.experimental.categories.Category;
64
65 @Category(LargeTests.class)
66 public class TestPerTableCFReplication {
67
68 private static final Log LOG = LogFactory.getLog(TestPerTableCFReplication.class);
69
70 private static Configuration conf1;
71 private static Configuration conf2;
72 private static Configuration conf3;
73
74 private static HBaseTestingUtility utility1;
75 private static HBaseTestingUtility utility2;
76 private static HBaseTestingUtility utility3;
77 private static final long SLEEP_TIME = 500;
78 private static final int NB_RETRIES = 100;
79
80 private static final TableName tableName = TableName.valueOf("test");
81 private static final TableName tabAName = TableName.valueOf("TA");
82 private static final TableName tabBName = TableName.valueOf("TB");
83 private static final TableName tabCName = TableName.valueOf("TC");
84 private static final byte[] famName = Bytes.toBytes("f");
85 private static final byte[] f1Name = Bytes.toBytes("f1");
86 private static final byte[] f2Name = Bytes.toBytes("f2");
87 private static final byte[] f3Name = Bytes.toBytes("f3");
88 private static final byte[] row1 = Bytes.toBytes("row1");
89 private static final byte[] row2 = Bytes.toBytes("row2");
90 private static final byte[] noRepfamName = Bytes.toBytes("norep");
91 private static final byte[] val = Bytes.toBytes("myval");
92
93 private static HTableDescriptor table;
94 private static HTableDescriptor tabA;
95 private static HTableDescriptor tabB;
96 private static HTableDescriptor tabC;
97
98 @BeforeClass
99 public static void setUpBeforeClass() throws Exception {
100 conf1 = HBaseConfiguration.create();
101 conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
102
103
104 conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
105 conf1.setInt("replication.source.size.capacity", 1024);
106 conf1.setLong("replication.source.sleepforretries", 100);
107 conf1.setInt("hbase.regionserver.maxlogs", 10);
108 conf1.setLong("hbase.master.logcleaner.ttl", 10);
109 conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
110 conf1.setBoolean("dfs.support.append", true);
111 conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
112 conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
113 "org.apache.hadoop.hbase.replication.TestMasterReplication$CoprocessorCounter");
114
115 utility1 = new HBaseTestingUtility(conf1);
116 utility1.startMiniZKCluster();
117 MiniZooKeeperCluster miniZK = utility1.getZkCluster();
118 new ZooKeeperWatcher(conf1, "cluster1", null, true);
119
120 conf2 = new Configuration(conf1);
121 conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
122
123 conf3 = new Configuration(conf1);
124 conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
125
126 utility2 = new HBaseTestingUtility(conf2);
127 utility2.setZkCluster(miniZK);
128 new ZooKeeperWatcher(conf2, "cluster3", null, true);
129
130 utility3 = new HBaseTestingUtility(conf3);
131 utility3.setZkCluster(miniZK);
132 new ZooKeeperWatcher(conf3, "cluster3", null, true);
133
134 table = new HTableDescriptor(tableName);
135 HColumnDescriptor fam = new HColumnDescriptor(famName);
136 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
137 table.addFamily(fam);
138 fam = new HColumnDescriptor(noRepfamName);
139 table.addFamily(fam);
140
141 tabA = new HTableDescriptor(tabAName);
142 fam = new HColumnDescriptor(f1Name);
143 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
144 tabA.addFamily(fam);
145 fam = new HColumnDescriptor(f2Name);
146 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
147 tabA.addFamily(fam);
148 fam = new HColumnDescriptor(f3Name);
149 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
150 tabA.addFamily(fam);
151
152 tabB = new HTableDescriptor(tabBName);
153 fam = new HColumnDescriptor(f1Name);
154 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
155 tabB.addFamily(fam);
156 fam = new HColumnDescriptor(f2Name);
157 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
158 tabB.addFamily(fam);
159 fam = new HColumnDescriptor(f3Name);
160 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
161 tabB.addFamily(fam);
162
163 tabC = new HTableDescriptor(tabCName);
164 fam = new HColumnDescriptor(f1Name);
165 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
166 tabC.addFamily(fam);
167 fam = new HColumnDescriptor(f2Name);
168 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
169 tabC.addFamily(fam);
170 fam = new HColumnDescriptor(f3Name);
171 fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
172 tabC.addFamily(fam);
173
174 utility1.startMiniCluster();
175 utility2.startMiniCluster();
176 utility3.startMiniCluster();
177 }
178
179 @AfterClass
180 public static void tearDownAfterClass() throws Exception {
181 utility3.shutdownMiniCluster();
182 utility2.shutdownMiniCluster();
183 utility1.shutdownMiniCluster();
184 }
185
186 @Test
187 public void testParseTableCFsFromConfig() {
188 Map<TableName, List<String>> tabCFsMap = null;
189
190
191 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig(null);
192 assertEquals(null, tabCFsMap);
193
194 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig("");
195 assertEquals(null, tabCFsMap);
196
197 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig(" ");
198 assertEquals(null, tabCFsMap);
199
200 TableName tab1 = TableName.valueOf("tab1");
201 TableName tab2 = TableName.valueOf("tab2");
202 TableName tab3 = TableName.valueOf("tab3");
203
204
205 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig("tab1");
206 assertEquals(1, tabCFsMap.size());
207 assertTrue(tabCFsMap.containsKey(tab1));
208 assertFalse(tabCFsMap.containsKey(tab2));
209 assertEquals(null, tabCFsMap.get(tab1));
210
211 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig("tab2:cf1");
212 assertEquals(1, tabCFsMap.size());
213 assertTrue(tabCFsMap.containsKey(tab2));
214 assertFalse(tabCFsMap.containsKey(tab1));
215 assertEquals(1, tabCFsMap.get(tab2).size());
216 assertEquals("cf1", tabCFsMap.get(tab2).get(0));
217
218 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig("tab3 : cf1 , cf3");
219 assertEquals(1, tabCFsMap.size());
220 assertTrue(tabCFsMap.containsKey(tab3));
221 assertFalse(tabCFsMap.containsKey(tab1));
222 assertEquals(2, tabCFsMap.get(tab3).size());
223 assertTrue(tabCFsMap.get(tab3).contains("cf1"));
224 assertTrue(tabCFsMap.get(tab3).contains("cf3"));
225
226
227 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig("tab1 ; tab2:cf1 ; tab3:cf1,cf3");
228
229 assertEquals(3, tabCFsMap.size());
230 assertTrue(tabCFsMap.containsKey(tab1));
231 assertTrue(tabCFsMap.containsKey(tab2));
232 assertTrue(tabCFsMap.containsKey(tab3));
233
234 assertEquals(null, tabCFsMap.get(tab1));
235
236 assertEquals(1, tabCFsMap.get(tab2).size());
237 assertEquals("cf1", tabCFsMap.get(tab2).get(0));
238
239 assertEquals(2, tabCFsMap.get(tab3).size());
240 assertTrue(tabCFsMap.get(tab3).contains("cf1"));
241 assertTrue(tabCFsMap.get(tab3).contains("cf3"));
242
243
244
245 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig(
246 "tab1 ; ; tab2:cf1 ; tab3:cf1,,cf3 ;");
247
248 assertEquals(3, tabCFsMap.size());
249 assertTrue(tabCFsMap.containsKey(tab1));
250 assertTrue(tabCFsMap.containsKey(tab2));
251 assertTrue(tabCFsMap.containsKey(tab3));
252
253 assertEquals(null, tabCFsMap.get(tab1));
254
255 assertEquals(1, tabCFsMap.get(tab2).size());
256 assertEquals("cf1", tabCFsMap.get(tab2).get(0));
257
258 assertEquals(2, tabCFsMap.get(tab3).size());
259 assertTrue(tabCFsMap.get(tab3).contains("cf1"));
260 assertTrue(tabCFsMap.get(tab3).contains("cf3"));
261
262
263
264 tabCFsMap = ReplicationSerDeHelper.parseTableCFsFromConfig(
265 "tab1:tt:cf1 ; tab2::cf1 ; tab3:cf1,cf3");
266
267 assertEquals(1, tabCFsMap.size());
268 assertFalse(tabCFsMap.containsKey(tab1));
269 assertFalse(tabCFsMap.containsKey(tab2));
270 assertTrue(tabCFsMap.containsKey(tab3));
271
272 assertEquals(2, tabCFsMap.get(tab3).size());
273 assertTrue(tabCFsMap.get(tab3).contains("cf1"));
274 assertTrue(tabCFsMap.get(tab3).contains("cf3"));
275 }
276
277 @Test
278 public void testTableCFsHelperConverter() {
279
280 ZooKeeperProtos.TableCF[] tableCFs = null;
281 Map<TableName, List<String>> tabCFsMap = null;
282
283
284 assertNull(ReplicationSerDeHelper.convert(tabCFsMap));
285
286 tabCFsMap = new HashMap<TableName, List<String>>();
287 tableCFs = ReplicationSerDeHelper.convert(tabCFsMap);
288 assertEquals(0, tableCFs.length);
289
290 TableName tab1 = TableName.valueOf("tab1");
291 TableName tab2 = TableName.valueOf("tab2");
292 TableName tab3 = TableName.valueOf("tab3");
293
294
295 tabCFsMap.clear();
296 tabCFsMap.put(tab1, null);
297 tableCFs = ReplicationSerDeHelper.convert(tabCFsMap);
298 assertEquals(1, tableCFs.length);
299 assertEquals(tab1.toString(),
300 tableCFs[0].getTableName().getQualifier().toStringUtf8());
301 assertEquals(0, tableCFs[0].getFamiliesCount());
302
303 tabCFsMap.clear();
304 tabCFsMap.put(tab2, new ArrayList<String>());
305 tabCFsMap.get(tab2).add("cf1");
306 tableCFs = ReplicationSerDeHelper.convert(tabCFsMap);
307 assertEquals(1, tableCFs.length);
308 assertEquals(tab2.toString(),
309 tableCFs[0].getTableName().getQualifier().toStringUtf8());
310 assertEquals(1, tableCFs[0].getFamiliesCount());
311 assertEquals("cf1", tableCFs[0].getFamilies(0).toStringUtf8());
312
313 tabCFsMap.clear();
314 tabCFsMap.put(tab3, new ArrayList<String>());
315 tabCFsMap.get(tab3).add("cf1");
316 tabCFsMap.get(tab3).add("cf3");
317 tableCFs = ReplicationSerDeHelper.convert(tabCFsMap);
318 assertEquals(1, tableCFs.length);
319 assertEquals(tab3.toString(),
320 tableCFs[0].getTableName().getQualifier().toStringUtf8());
321 assertEquals(2, tableCFs[0].getFamiliesCount());
322 assertEquals("cf1", tableCFs[0].getFamilies(0).toStringUtf8());
323 assertEquals("cf3", tableCFs[0].getFamilies(1).toStringUtf8());
324
325 tabCFsMap.clear();
326 tabCFsMap.put(tab1, null);
327 tabCFsMap.put(tab2, new ArrayList<String>());
328 tabCFsMap.get(tab2).add("cf1");
329 tabCFsMap.put(tab3, new ArrayList<String>());
330 tabCFsMap.get(tab3).add("cf1");
331 tabCFsMap.get(tab3).add("cf3");
332
333 tableCFs = ReplicationSerDeHelper.convert(tabCFsMap);
334 assertEquals(3, tableCFs.length);
335 assertNotNull(ReplicationSerDeHelper.getTableCF(tableCFs, tab1.toString()));
336 assertNotNull(ReplicationSerDeHelper.getTableCF(tableCFs, tab2.toString()));
337 assertNotNull(ReplicationSerDeHelper.getTableCF(tableCFs, tab3.toString()));
338
339 assertEquals(0,
340 ReplicationSerDeHelper.getTableCF(tableCFs, tab1.toString()).getFamiliesCount());
341
342 assertEquals(1,
343 ReplicationSerDeHelper.getTableCF(tableCFs, tab2.toString()).getFamiliesCount());
344 assertEquals("cf1",
345 ReplicationSerDeHelper.getTableCF(tableCFs, tab2.toString()).getFamilies(0).toStringUtf8());
346
347 assertEquals(2,
348 ReplicationSerDeHelper.getTableCF(tableCFs, tab3.toString()).getFamiliesCount());
349 assertEquals("cf1",
350 ReplicationSerDeHelper.getTableCF(tableCFs, tab3.toString()).getFamilies(0).toStringUtf8());
351 assertEquals("cf3",
352 ReplicationSerDeHelper.getTableCF(tableCFs, tab3.toString()).getFamilies(1).toStringUtf8());
353
354 tabCFsMap = ReplicationSerDeHelper.convert2Map(tableCFs);
355 assertEquals(3, tabCFsMap.size());
356 assertTrue(tabCFsMap.containsKey(tab1));
357 assertTrue(tabCFsMap.containsKey(tab2));
358 assertTrue(tabCFsMap.containsKey(tab3));
359
360 assertEquals(null, tabCFsMap.get(tab1));
361
362 assertEquals(1, tabCFsMap.get(tab2).size());
363 assertEquals("cf1", tabCFsMap.get(tab2).get(0));
364
365 assertEquals(2, tabCFsMap.get(tab3).size());
366 assertTrue(tabCFsMap.get(tab3).contains("cf1"));
367 assertTrue(tabCFsMap.get(tab3).contains("cf3"));
368 }
369
370 @Test(timeout=300000)
371 public void testPerTableCFReplication() throws Exception {
372 LOG.info("testPerTableCFReplication");
373 ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf1);
374 Connection connection1 = ConnectionFactory.createConnection(conf1);
375 Connection connection2 = ConnectionFactory.createConnection(conf2);
376 Connection connection3 = ConnectionFactory.createConnection(conf3);
377 try {
378 Admin admin1 = connection1.getAdmin();
379 Admin admin2 = connection2.getAdmin();
380 Admin admin3 = connection3.getAdmin();
381
382 admin1.createTable(tabA);
383 admin1.createTable(tabB);
384 admin1.createTable(tabC);
385 admin2.createTable(tabA);
386 admin2.createTable(tabB);
387 admin2.createTable(tabC);
388 admin3.createTable(tabA);
389 admin3.createTable(tabB);
390 admin3.createTable(tabC);
391
392 Table htab1A = connection1.getTable(tabAName);
393 Table htab2A = connection2.getTable(tabAName);
394 Table htab3A = connection3.getTable(tabAName);
395
396 Table htab1B = connection1.getTable(tabBName);
397 Table htab2B = connection2.getTable(tabBName);
398 Table htab3B = connection3.getTable(tabBName);
399
400 Table htab1C = connection1.getTable(tabCName);
401 Table htab2C = connection2.getTable(tabCName);
402 Table htab3C = connection3.getTable(tabCName);
403
404
405 ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
406 rpc2.setClusterKey(utility2.getClusterKey());
407 Map<TableName, List<String>> tableCFs = new HashMap<>();
408 tableCFs.put(tabCName, null);
409 tableCFs.put(tabBName, new ArrayList<String>());
410 tableCFs.get(tabBName).add("f1");
411 tableCFs.get(tabBName).add("f3");
412 replicationAdmin.addPeer("2", rpc2, tableCFs);
413
414 ReplicationPeerConfig rpc3 = new ReplicationPeerConfig();
415 rpc3.setClusterKey(utility3.getClusterKey());
416 tableCFs.clear();
417 tableCFs.put(tabAName, null);
418 tableCFs.put(tabBName, new ArrayList<String>());
419 tableCFs.get(tabBName).add("f1");
420 tableCFs.get(tabBName).add("f2");
421 replicationAdmin.addPeer("3", rpc3, tableCFs);
422
423
424 putAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
425 ensureRowNotReplicated(row1, f1Name, htab2A);
426 deleteAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
427
428 putAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
429 ensureRowNotReplicated(row1, f2Name, htab2A);
430 deleteAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
431
432 putAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
433 ensureRowNotReplicated(row1, f3Name, htab2A);
434 deleteAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
435
436
437 putAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
438 deleteAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
439
440
441 putAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
442 ensureRowNotReplicated(row1, f2Name, htab2B);
443 deleteAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
444
445
446 putAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
447 ensureRowNotReplicated(row1, f3Name, htab3B);
448 deleteAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
449
450
451 putAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
452 ensureRowNotReplicated(row1, f1Name, htab3C);
453 deleteAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
454
455 putAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
456 ensureRowNotReplicated(row1, f2Name, htab3C);
457 deleteAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
458
459 putAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
460 ensureRowNotReplicated(row1, f3Name, htab3C);
461 deleteAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
462
463
464 tableCFs.clear();
465 tableCFs.put(tabAName, new ArrayList<String>());
466 tableCFs.get(tabAName).add("f1");
467 tableCFs.get(tabAName).add("f2");
468 tableCFs.put(tabCName, new ArrayList<String>());
469 tableCFs.get(tabCName).add("f2");
470 tableCFs.get(tabCName).add("f3");
471 replicationAdmin.setPeerTableCFs("2", tableCFs);
472
473 tableCFs.clear();
474 tableCFs.put(tabBName, null);
475 tableCFs.put(tabCName, new ArrayList<String>());
476 tableCFs.get(tabCName).add("f3");
477 replicationAdmin.setPeerTableCFs("3", tableCFs);
478
479
480 putAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
481 ensureRowNotReplicated(row2, f1Name, htab3A);
482 deleteAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
483
484 putAndWaitWithFamily(row2, f2Name, htab1A, htab2A);
485 ensureRowNotReplicated(row2, f2Name, htab3A);
486 deleteAndWaitWithFamily(row2, f2Name, htab1A, htab2A);
487
488 putAndWaitWithFamily(row2, f3Name, htab1A);
489 ensureRowNotReplicated(row2, f3Name, htab2A, htab3A);
490 deleteAndWaitWithFamily(row2, f3Name, htab1A);
491
492
493 putAndWaitWithFamily(row2, f1Name, htab1B, htab3B);
494 ensureRowNotReplicated(row2, f1Name, htab2B);
495 deleteAndWaitWithFamily(row2, f1Name, htab1B, htab3B);
496
497 putAndWaitWithFamily(row2, f2Name, htab1B, htab3B);
498 ensureRowNotReplicated(row2, f2Name, htab2B);
499 deleteAndWaitWithFamily(row2, f2Name, htab1B, htab3B);
500
501 putAndWaitWithFamily(row2, f3Name, htab1B, htab3B);
502 ensureRowNotReplicated(row2, f3Name, htab2B);
503 deleteAndWaitWithFamily(row2, f3Name, htab1B, htab3B);
504
505
506 putAndWaitWithFamily(row2, f1Name, htab1C);
507 ensureRowNotReplicated(row2, f1Name, htab2C, htab3C);
508 deleteAndWaitWithFamily(row2, f1Name, htab1C);
509
510 putAndWaitWithFamily(row2, f2Name, htab1C, htab2C);
511 ensureRowNotReplicated(row2, f2Name, htab3C);
512 deleteAndWaitWithFamily(row2, f2Name, htab1C, htab2C);
513
514 putAndWaitWithFamily(row2, f3Name, htab1C, htab2C, htab3C);
515 deleteAndWaitWithFamily(row2, f3Name, htab1C, htab2C, htab3C);
516 } finally {
517 connection1.close();
518 connection2.close();
519 connection3.close();
520 }
521 }
522
523 private void ensureRowNotReplicated(byte[] row, byte[] fam, Table... tables) throws IOException {
524 Get get = new Get(row);
525 get.addFamily(fam);
526 for (Table table : tables) {
527 Result res = table.get(get);
528 assertEquals(0, res.size());
529 }
530 }
531
532 private void deleteAndWaitWithFamily(byte[] row, byte[] fam,
533 Table source, Table... targets)
534 throws Exception {
535 Delete del = new Delete(row);
536 del.deleteFamily(fam);
537 source.delete(del);
538
539 Get get = new Get(row);
540 get.addFamily(fam);
541 for (int i = 0; i < NB_RETRIES; i++) {
542 if (i==NB_RETRIES-1) {
543 fail("Waited too much time for del replication");
544 }
545 boolean removedFromAll = true;
546 for (Table target : targets) {
547 Result res = target.get(get);
548 if (res.size() >= 1) {
549 LOG.info("Row not deleted");
550 removedFromAll = false;
551 break;
552 }
553 }
554 if (removedFromAll) {
555 break;
556 } else {
557 Thread.sleep(SLEEP_TIME);
558 }
559 }
560 }
561
562 private void putAndWaitWithFamily(byte[] row, byte[] fam,
563 Table source, Table... targets)
564 throws Exception {
565 Put put = new Put(row);
566 put.add(fam, row, val);
567 source.put(put);
568
569 Get get = new Get(row);
570 get.addFamily(fam);
571 for (int i = 0; i < NB_RETRIES; i++) {
572 if (i==NB_RETRIES-1) {
573 fail("Waited too much time for put replication");
574 }
575 boolean replicatedToAll = true;
576 for (Table target : targets) {
577 Result res = target.get(get);
578 if (res.size() == 0) {
579 LOG.info("Row not available");
580 replicatedToAll = false;
581 break;
582 } else {
583 assertEquals(res.size(), 1);
584 assertArrayEquals(res.value(), val);
585 }
586 }
587 if (replicatedToAll) {
588 break;
589 } else {
590 Thread.sleep(SLEEP_TIME);
591 }
592 }
593 }
594 }