1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertNull;
24 import static org.mockito.Mockito.mock;
25 import static org.mockito.Mockito.when;
26
27 import com.google.common.collect.Lists;
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.concurrent.Executors;
31 import java.util.concurrent.atomic.AtomicLong;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.commons.logging.impl.Log4JLogger;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.hbase.Cell;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionLocation;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.KeyValue;
43 import org.apache.hadoop.hbase.KeyValue.Type;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.Waiter;
46 import org.apache.hadoop.hbase.client.ClusterConnection;
47 import org.apache.hadoop.hbase.client.Connection;
48 import org.apache.hadoop.hbase.client.ConnectionFactory;
49 import org.apache.hadoop.hbase.client.RegionLocator;
50 import org.apache.hadoop.hbase.client.RpcRetryingCaller;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
53 import org.apache.hadoop.hbase.regionserver.HRegionServer;
54 import org.apache.hadoop.hbase.regionserver.Region;
55 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
56 import org.apache.hadoop.hbase.replication.ReplicationException;
57 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
58 import org.apache.hadoop.hbase.testclassification.MediumTests;
59 import org.apache.hadoop.hbase.util.Bytes;
60 import org.apache.hadoop.hbase.util.FSTableDescriptors;
61 import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
62 import org.apache.hadoop.hbase.wal.WAL.Entry;
63 import org.apache.hadoop.hbase.wal.WALKey;
64 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
65 import org.apache.log4j.Level;
66 import org.junit.AfterClass;
67 import org.junit.BeforeClass;
68 import org.junit.Test;
69 import org.junit.experimental.categories.Category;
70
71
72
73
74
75 @Category(MediumTests.class)
76 public class TestRegionReplicaReplicationEndpoint {
77
78 private static final Log LOG = LogFactory.getLog(TestRegionReplicaReplicationEndpoint.class);
79
80 static {
81 ((Log4JLogger)RpcRetryingCaller.LOG).getLogger().setLevel(Level.ALL);
82 }
83
84 private static final int NB_SERVERS = 2;
85
86 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
87
88 @BeforeClass
89 public static void beforeClass() throws Exception {
90 Configuration conf = HTU.getConfiguration();
91 conf.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
92 conf.setInt("replication.source.size.capacity", 10240);
93 conf.setLong("replication.source.sleepforretries", 100);
94 conf.setInt("hbase.regionserver.maxlogs", 10);
95 conf.setLong("hbase.master.logcleaner.ttl", 10);
96 conf.setInt("zookeeper.recovery.retry", 1);
97 conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
98 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
99 conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true);
100 conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
101 conf.setInt("replication.stats.thread.period.seconds", 5);
102 conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
103 conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
104 conf.setInt("hbase.client.serverside.retries.multiplier", 1);
105
106 HTU.startMiniCluster(NB_SERVERS);
107 }
108
109 @AfterClass
110 public static void afterClass() throws Exception {
111 HTU.shutdownMiniCluster();
112 }
113
114 @Test
115 public void testRegionReplicaReplicationPeerIsCreated() throws IOException, ReplicationException {
116
117
118 ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
119 String peerId = "region_replica_replication";
120
121 if (admin.getPeerConfig(peerId) != null) {
122 admin.removePeer(peerId);
123 }
124
125 HTableDescriptor htd = HTU.createTableDescriptor(
126 "testReplicationPeerIsCreated_no_region_replicas");
127 HTU.getHBaseAdmin().createTable(htd);
128 ReplicationPeerConfig peerConfig = admin.getPeerConfig(peerId);
129 assertNull(peerConfig);
130
131 htd = HTU.createTableDescriptor("testReplicationPeerIsCreated");
132 htd.setRegionReplication(2);
133 HTU.getHBaseAdmin().createTable(htd);
134
135
136 peerConfig = admin.getPeerConfig(peerId);
137 assertNotNull(peerConfig);
138 assertEquals(peerConfig.getClusterKey(), ZKConfig.getZooKeeperClusterKey(
139 HTU.getConfiguration()));
140 assertEquals(peerConfig.getReplicationEndpointImpl(),
141 RegionReplicaReplicationEndpoint.class.getName());
142 admin.close();
143 }
144
145 @Test (timeout=240000)
146 public void testRegionReplicaReplicationPeerIsCreatedForModifyTable() throws Exception {
147
148
149 ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
150 String peerId = "region_replica_replication";
151
152 if (admin.getPeerConfig(peerId) != null) {
153 admin.removePeer(peerId);
154 }
155
156 HTableDescriptor htd
157 = HTU.createTableDescriptor("testRegionReplicaReplicationPeerIsCreatedForModifyTable");
158 HTU.getHBaseAdmin().createTable(htd);
159
160
161 ReplicationPeerConfig peerConfig = admin.getPeerConfig(peerId);
162 assertNull(peerConfig);
163
164 HTU.getHBaseAdmin().disableTable(htd.getTableName());
165 htd.setRegionReplication(2);
166 HTU.getHBaseAdmin().modifyTable(htd.getTableName(), htd);
167 HTU.getHBaseAdmin().enableTable(htd.getTableName());
168
169
170 peerConfig = admin.getPeerConfig(peerId);
171 assertNotNull(peerConfig);
172 assertEquals(peerConfig.getClusterKey(), ZKConfig.getZooKeeperClusterKey(
173 HTU.getConfiguration()));
174 assertEquals(peerConfig.getReplicationEndpointImpl(),
175 RegionReplicaReplicationEndpoint.class.getName());
176 admin.close();
177 }
178
179 public void testRegionReplicaReplication(int regionReplication) throws Exception {
180
181
182 TableName tableName = TableName.valueOf("testRegionReplicaReplicationWithReplicas_"
183 + regionReplication);
184 HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
185 htd.setRegionReplication(regionReplication);
186 HTU.getHBaseAdmin().createTable(htd);
187 TableName tableNameNoReplicas =
188 TableName.valueOf("testRegionReplicaReplicationWithReplicas_NO_REPLICAS");
189 HTU.deleteTableIfAny(tableNameNoReplicas);
190 HTU.createTable(tableNameNoReplicas, HBaseTestingUtility.fam1);
191
192 Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
193 Table table = connection.getTable(tableName);
194 Table tableNoReplicas = connection.getTable(tableNameNoReplicas);
195
196 try {
197
198 HTU.loadNumericRows(tableNoReplicas, HBaseTestingUtility.fam1, 6000, 7000);
199
200
201 HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);
202
203 verifyReplication(tableName, regionReplication, 0, 1000);
204
205 } finally {
206 table.close();
207 tableNoReplicas.close();
208 HTU.deleteTableIfAny(tableNameNoReplicas);
209 connection.close();
210 }
211 }
212
213 private void verifyReplication(TableName tableName, int regionReplication,
214 final int startRow, final int endRow) throws Exception {
215 verifyReplication(tableName, regionReplication, startRow, endRow, true);
216 }
217
218 private void verifyReplication(TableName tableName, int regionReplication,
219 final int startRow, final int endRow, final boolean present) throws Exception {
220
221 final Region[] regions = new Region[regionReplication];
222
223 for (int i=0; i < NB_SERVERS; i++) {
224 HRegionServer rs = HTU.getMiniHBaseCluster().getRegionServer(i);
225 List<Region> onlineRegions = rs.getOnlineRegions(tableName);
226 for (Region region : onlineRegions) {
227 regions[region.getRegionInfo().getReplicaId()] = region;
228 }
229 }
230
231 for (Region region : regions) {
232 assertNotNull(region);
233 }
234
235 for (int i = 1; i < regionReplication; i++) {
236 final Region region = regions[i];
237
238 Waiter.waitFor(HTU.getConfiguration(), 90000, 1000, new Waiter.Predicate<Exception>() {
239 @Override
240 public boolean evaluate() throws Exception {
241 LOG.info("verifying replication for region replica:" + region.getRegionInfo());
242 try {
243 HTU.verifyNumericRows(region, HBaseTestingUtility.fam1, startRow, endRow, present);
244 } catch(Throwable ex) {
245 LOG.warn("Verification from secondary region is not complete yet", ex);
246
247 return false;
248 }
249 return true;
250 }
251 });
252 }
253 }
254
255 @Test(timeout = 240000)
256 public void testRegionReplicaReplicationWith2Replicas() throws Exception {
257 testRegionReplicaReplication(2);
258 }
259
260 @Test(timeout = 240000)
261 public void testRegionReplicaReplicationWith3Replicas() throws Exception {
262 testRegionReplicaReplication(3);
263 }
264
265 @Test(timeout = 240000)
266 public void testRegionReplicaReplicationWith10Replicas() throws Exception {
267 testRegionReplicaReplication(10);
268 }
269
270 @Test (timeout = 240000)
271 public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
272 int regionReplication = 3;
273 TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
274 HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
275 htd.setRegionReplication(regionReplication);
276 htd.setRegionMemstoreReplication(false);
277 HTU.getHBaseAdmin().createTable(htd);
278
279 Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
280 Table table = connection.getTable(tableName);
281 try {
282
283 final int STEP = 100;
284 for (int i = 0; i < 3; ++i) {
285 final int startRow = i * STEP;
286 final int endRow = (i + 1) * STEP;
287 LOG.info("Writing data from " + startRow + " to " + endRow);
288 HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
289 verifyReplication(tableName, regionReplication, startRow, endRow, false);
290
291
292 LOG.info("flushing table");
293 HTU.flush(tableName);
294 verifyReplication(tableName, regionReplication, 0, endRow, true);
295 }
296 } finally {
297 table.close();
298 connection.close();
299 }
300 }
301
302 @Test (timeout = 240000)
303 public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception {
304
305
306
307
308 int regionReplication = 3;
309 TableName tableName = TableName.valueOf("testRegionReplicaReplicationForFlushAndCompaction");
310 HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
311 htd.setRegionReplication(regionReplication);
312 HTU.getHBaseAdmin().createTable(htd);
313
314 Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
315 Table table = connection.getTable(tableName);
316 try {
317
318
319 for (int i = 0; i < 6000; i += 1000) {
320 LOG.info("Writing data from " + i + " to " + (i+1000));
321 HTU.loadNumericRows(table, HBaseTestingUtility.fam1, i, i+1000);
322 LOG.info("flushing table");
323 HTU.flush(tableName);
324 LOG.info("compacting table");
325 HTU.compact(tableName, false);
326 }
327
328 verifyReplication(tableName, regionReplication, 0, 1000);
329 } finally {
330 table.close();
331 connection.close();
332 }
333 }
334
335 @Test(timeout = 240000)
336 public void testRegionReplicaReplicationIgnoresDisabledTables() throws Exception {
337 testRegionReplicaReplicationIgnores(false, false);
338 }
339
340 @Test(timeout = 240000)
341 public void testRegionReplicaReplicationIgnoresDroppedTables() throws Exception {
342 testRegionReplicaReplicationIgnores(true, false);
343 }
344
345 @Test(timeout = 240000)
346 public void testRegionReplicaReplicationIgnoresNonReplicatedTables() throws Exception {
347 testRegionReplicaReplicationIgnores(false, true);
348 }
349
350 public void testRegionReplicaReplicationIgnores(boolean dropTable, boolean disableReplication)
351 throws Exception {
352
353
354
355 TableName tableName = TableName.valueOf("testRegionReplicaReplicationIgnoresDisabledTables"
356 + "_drop_" + dropTable + "_disabledReplication_" + disableReplication);
357 HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
358 int regionReplication = 3;
359 htd.setRegionReplication(regionReplication);
360 HTU.deleteTableIfAny(tableName);
361 HTU.getHBaseAdmin().createTable(htd);
362 TableName toBeDisabledTable = TableName.valueOf(
363 dropTable ? "droppedTable" : (disableReplication ? "disableReplication" : "disabledTable"));
364 HTU.deleteTableIfAny(toBeDisabledTable);
365 htd = HTU.createTableDescriptor(toBeDisabledTable.toString());
366 htd.setRegionReplication(regionReplication);
367 HTU.getHBaseAdmin().createTable(htd);
368
369
370 ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
371 admin.disablePeer(ServerRegionReplicaUtil.getReplicationPeerId());
372
373
374
375 Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
376 Table table = connection.getTable(tableName);
377 Table tableToBeDisabled = connection.getTable(toBeDisabledTable);
378
379 HTU.loadNumericRows(tableToBeDisabled, HBaseTestingUtility.fam1, 6000, 7000);
380
381 AtomicLong skippedEdits = new AtomicLong();
382 RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink =
383 mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class);
384 when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits);
385 FSTableDescriptors fstd = new FSTableDescriptors(HTU.getConfiguration(),
386 FileSystem.get(HTU.getConfiguration()), HTU.getDefaultRootDirPath());
387 RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter =
388 new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink,
389 (ClusterConnection) connection,
390 Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, fstd);
391
392 RegionLocator rl = connection.getRegionLocator(toBeDisabledTable);
393 HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY);
394 byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes();
395 Entry entry = new Entry(
396 new WALKey(encodedRegionName, toBeDisabledTable, 1),
397 new WALEdit());
398
399 HTU.getHBaseAdmin().disableTable(toBeDisabledTable);
400 if (dropTable) {
401 HTU.getHBaseAdmin().deleteTable(toBeDisabledTable);
402 } else if (disableReplication) {
403 htd.setRegionReplication(regionReplication - 2);
404 HTU.getHBaseAdmin().modifyTable(toBeDisabledTable, htd);
405 HTU.getHBaseAdmin().enableTable(toBeDisabledTable);
406 }
407
408 sinkWriter.append(toBeDisabledTable, encodedRegionName,
409 HConstants.EMPTY_BYTE_ARRAY, Lists.newArrayList(entry, entry));
410
411 assertEquals(2, skippedEdits.get());
412 if (disableReplication) {
413
414 HTU.getHBaseAdmin().disableTable(toBeDisabledTable);
415 htd.setRegionReplication(regionReplication);
416 HTU.getHBaseAdmin().modifyTable(toBeDisabledTable, htd);
417 HTU.getHBaseAdmin().enableTable(toBeDisabledTable);
418 }
419 try {
420
421
422
423 HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);
424
425
426 admin.enablePeer(ServerRegionReplicaUtil.getReplicationPeerId());
427
428 verifyReplication(tableName, regionReplication, 0, 1000);
429
430 } finally {
431 admin.close();
432 table.close();
433 rl.close();
434 tableToBeDisabled.close();
435 HTU.deleteTableIfAny(toBeDisabledTable);
436 connection.close();
437 }
438 }
439 }