1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.*;
22 import static org.mockito.Matchers.any;
23 import static org.mockito.Matchers.anyBoolean;
24 import static org.mockito.Mockito.mock;
25 import static org.mockito.Mockito.spy;
26 import static org.mockito.Mockito.times;
27 import static org.mockito.Mockito.verify;
28 import static org.mockito.Mockito.when;
29 import static org.apache.hadoop.hbase.regionserver.TestHRegion.*;
30
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.Random;
37 import java.util.UUID;
38
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.fs.FSDataOutputStream;
43 import org.apache.hadoop.fs.FileSystem;
44 import org.apache.hadoop.fs.Path;
45 import org.apache.hadoop.hbase.Cell;
46 import org.apache.hadoop.hbase.CellUtil;
47 import org.apache.hadoop.hbase.HBaseTestingUtility;
48 import org.apache.hadoop.hbase.HColumnDescriptor;
49 import org.apache.hadoop.hbase.HConstants;
50 import org.apache.hadoop.hbase.HRegionInfo;
51 import org.apache.hadoop.hbase.HTableDescriptor;
52 import org.apache.hadoop.hbase.KeyValue;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.client.Durability;
56 import org.apache.hadoop.hbase.client.Get;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.executor.ExecutorService;
59 import org.apache.hadoop.hbase.io.hfile.HFile;
60 import org.apache.hadoop.hbase.io.hfile.HFileContext;
61 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
62 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
63 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
64 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
65 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
66 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;
67 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
68 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
69 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
70 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
71 import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
72 import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
73 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
74 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
75 import org.apache.hadoop.hbase.testclassification.MediumTests;
76 import org.apache.hadoop.hbase.util.Bytes;
77 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
78 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
79 import org.apache.hadoop.hbase.util.FSUtils;
80 import org.apache.hadoop.hbase.util.Pair;
81 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
82 import org.apache.hadoop.hbase.wal.WAL;
83 import org.apache.hadoop.hbase.wal.WALFactory;
84 import org.apache.hadoop.hbase.wal.WALKey;
85 import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay;
86 import org.apache.hadoop.util.StringUtils;
87 import org.junit.After;
88 import org.junit.Before;
89 import org.junit.Rule;
90 import org.junit.Test;
91 import org.junit.experimental.categories.Category;
92 import org.junit.rules.TestName;
93
94 import com.google.common.collect.Lists;
95 import com.google.protobuf.ByteString;
96
97
98
99
100
101 @Category(MediumTests.class)
102 public class TestHRegionReplayEvents {
103
104 private static final Log LOG = LogFactory.getLog(TestHRegion.class);
105 @Rule public TestName name = new TestName();
106
107 private static HBaseTestingUtility TEST_UTIL;
108
109 public static Configuration CONF ;
110 private String dir;
111 private static FileSystem FILESYSTEM;
112
113 private byte[][] families = new byte[][] {
114 Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")};
115
116
117 protected byte[] tableName;
118 protected String method;
119 protected final byte[] row = Bytes.toBytes("rowA");
120 protected final byte[] row2 = Bytes.toBytes("rowB");
121 protected byte[] cq = Bytes.toBytes("cq");
122
123
124 private Path rootDir;
125 private HTableDescriptor htd;
126 private long time;
127 private RegionServerServices rss;
128 private HRegionInfo primaryHri, secondaryHri;
129 private HRegion primaryRegion, secondaryRegion;
130 private WALFactory wals;
131 private WAL walPrimary, walSecondary;
132 private WAL.Reader reader;
133
134 @Before
135 public void setup() throws IOException {
136 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
137 FILESYSTEM = TEST_UTIL.getTestFileSystem();
138 CONF = TEST_UTIL.getConfiguration();
139 dir = TEST_UTIL.getDataTestDir("TestHRegionReplayEvents").toString();
140 method = name.getMethodName();
141 tableName = Bytes.toBytes(name.getMethodName());
142 rootDir = new Path(dir + method);
143 TEST_UTIL.getConfiguration().set(HConstants.HBASE_DIR, rootDir.toString());
144 method = name.getMethodName();
145
146 htd = new HTableDescriptor(TableName.valueOf(method));
147 for (byte[] family : families) {
148 htd.addFamily(new HColumnDescriptor(family));
149 }
150
151 time = System.currentTimeMillis();
152
153 primaryHri = new HRegionInfo(htd.getTableName(),
154 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
155 false, time, 0);
156 secondaryHri = new HRegionInfo(htd.getTableName(),
157 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
158 false, time, 1);
159
160 wals = TestHRegion.createWALFactory(CONF, rootDir);
161 walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes(),
162 primaryHri.getTable().getNamespace());
163 walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes(),
164 secondaryHri.getTable().getNamespace());
165
166 rss = mock(RegionServerServices.class);
167 when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1));
168 when(rss.getConfiguration()).thenReturn(CONF);
169 when(rss.getRegionServerAccounting()).thenReturn(new RegionServerAccounting());
170 String string = org.apache.hadoop.hbase.executor.EventType.RS_COMPACTED_FILES_DISCHARGER
171 .toString();
172 ExecutorService es = new ExecutorService(string);
173 es.startExecutorService(
174 string+"-"+string, 1);
175 when(rss.getExecutorService()).thenReturn(es);
176 primaryRegion = HRegion.createHRegion(primaryHri, rootDir, CONF, htd, walPrimary);
177 primaryRegion.close();
178 List<Region> regions = new ArrayList<Region>();
179 regions.add(primaryRegion);
180 when(rss.getOnlineRegions()).thenReturn(regions);
181
182 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
183 secondaryRegion = HRegion.openHRegion(secondaryHri, htd, null, CONF, rss, null);
184
185 reader = null;
186 }
187
188 @After
189 public void tearDown() throws Exception {
190 if (reader != null) {
191 reader.close();
192 }
193
194 if (primaryRegion != null) {
195 HRegion.closeHRegion(primaryRegion);
196 }
197 if (secondaryRegion != null) {
198 HRegion.closeHRegion(secondaryRegion);
199 }
200
201 EnvironmentEdgeManagerTestHelper.reset();
202 LOG.info("Cleaning test directory: " + TEST_UTIL.getDataTestDir());
203 TEST_UTIL.cleanupTestDir();
204 }
205
206 String getName() {
207 return name.getMethodName();
208 }
209
210
211
212
213
214
215
216
217
218
219
220
221 @Test
222 public void testRegionReplicaSecondaryCannotFlush() throws IOException {
223
224
225
226 putDataByReplay(secondaryRegion, 0, 1000, cq, families);
227
228 verifyData(secondaryRegion, 0, 1000, cq, families);
229
230
231 FlushResultImpl flush = (FlushResultImpl)secondaryRegion.flush(true);
232 assertEquals(flush.result, FlushResultImpl.Result.CANNOT_FLUSH);
233
234 verifyData(secondaryRegion, 0, 1000, cq, families);
235
236
237 Map<byte[], List<StoreFile>> files = secondaryRegion.close(false);
238
239 for (List<StoreFile> f : files.values()) {
240 assertTrue(f.isEmpty());
241 }
242 }
243
244
245
246
247
248 @Test (timeout = 60000)
249 public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose() throws IOException {
250
251 int start = 0;
252 LOG.info("-- Writing some data to primary from " + start + " to " + (start+100));
253 putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
254 LOG.info("-- Flushing primary, creating 3 files for 3 stores");
255 primaryRegion.flush(true);
256
257
258 reader = createWALReaderForPrimary();
259
260 LOG.info("-- Replaying edits and flush events in secondary");
261 while (true) {
262 WAL.Entry entry = reader.next();
263 if (entry == null) {
264 break;
265 }
266 FlushDescriptor flushDesc
267 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
268 if (flushDesc != null) {
269 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
270 LOG.info("-- Replaying flush start in secondary");
271 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(flushDesc);
272 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
273 LOG.info("-- NOT Replaying flush commit in secondary");
274 }
275 } else {
276 replayEdit(secondaryRegion, entry);
277 }
278 }
279
280 assertTrue(rss.getRegionServerAccounting().getGlobalMemstoreSize() > 0);
281
282 secondaryRegion.close();
283
284
285 assertEquals(0, rss.getRegionServerAccounting().getGlobalMemstoreSize());
286 }
287
288 static int replayEdit(HRegion region, WAL.Entry entry) throws IOException {
289 if (WALEdit.isMetaEditFamily(entry.getEdit().getCells().get(0))) {
290 return 0;
291 }
292 Put put = new Put(entry.getEdit().getCells().get(0).getRow());
293 for (Cell cell : entry.getEdit().getCells()) put.add(cell);
294 put.setDurability(Durability.SKIP_WAL);
295 MutationReplay mutation = new MutationReplay(MutationType.PUT, put, 0, 0);
296 region.batchReplay(new MutationReplay[] {mutation},
297 entry.getKey().getLogSeqNum());
298 return Integer.parseInt(Bytes.toString(put.getRow()));
299 }
300
301 WAL.Reader createWALReaderForPrimary() throws FileNotFoundException, IOException {
302 return wals.createReader(TEST_UTIL.getTestFileSystem(),
303 DefaultWALProvider.getCurrentFileName(walPrimary),
304 TEST_UTIL.getConfiguration());
305 }
306
307 @Test
308 public void testReplayFlushesAndCompactions() throws IOException {
309
310
311
312 putDataWithFlushes(primaryRegion, 100, 300, 100);
313
314
315 LOG.info("-- Compacting primary, only 1 store");
316 primaryRegion.compactStore(Bytes.toBytes("cf1"),
317 NoLimitThroughputController.INSTANCE);
318
319
320 reader = createWALReaderForPrimary();
321
322 LOG.info("-- Replaying edits and flush events in secondary");
323 int lastReplayed = 0;
324 int expectedStoreFileCount = 0;
325 while (true) {
326 WAL.Entry entry = reader.next();
327 if (entry == null) {
328 break;
329 }
330 FlushDescriptor flushDesc
331 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
332 CompactionDescriptor compactionDesc
333 = WALEdit.getCompaction(entry.getEdit().getCells().get(0));
334 if (flushDesc != null) {
335
336 verifyData(secondaryRegion, 0, lastReplayed, cq, families);
337 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
338 long storeMemstoreSize = store.getMemStoreSize();
339 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
340 long storeFlushableSize = store.getFlushableSize();
341 long storeSize = store.getSize();
342 long storeSizeUncompressed = store.getStoreSizeUncompressed();
343 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
344 LOG.info("-- Replaying flush start in secondary");
345 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(flushDesc);
346 assertNull(result.result);
347 assertEquals(result.flushOpSeqId, flushDesc.getFlushSequenceNumber());
348
349
350 long newStoreMemstoreSize = store.getMemStoreSize();
351 LOG.info("Memstore size reduced by:"
352 + StringUtils.humanReadableInt(newStoreMemstoreSize - storeMemstoreSize));
353 assertTrue(storeMemstoreSize > newStoreMemstoreSize);
354
355 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
356 LOG.info("-- Replaying flush commit in secondary");
357 secondaryRegion.replayWALFlushCommitMarker(flushDesc);
358
359
360 expectedStoreFileCount++;
361 for (Store s : secondaryRegion.getStores()) {
362 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
363 }
364 long newFlushableSize = store.getFlushableSize();
365 assertTrue(storeFlushableSize > newFlushableSize);
366
367
368 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
369 assertTrue(regionMemstoreSize > newRegionMemstoreSize);
370
371
372 assertTrue(store.getSize() > storeSize);
373 assertTrue(store.getStoreSizeUncompressed() > storeSizeUncompressed);
374 assertEquals(store.getSize(), store.getStorefilesSize());
375 }
376
377 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
378 } else if (compactionDesc != null) {
379 secondaryRegion.replayWALCompactionMarker(compactionDesc, true, false, Long.MAX_VALUE);
380
381
382 for (Store store : secondaryRegion.getStores()) {
383 if (store.getColumnFamilyName().equals("cf1")) {
384 assertEquals(1, store.getStorefilesCount());
385 } else {
386 assertEquals(expectedStoreFileCount, store.getStorefilesCount());
387 }
388 }
389 } else {
390 lastReplayed = replayEdit(secondaryRegion, entry);;
391 }
392 }
393
394 assertEquals(400-1, lastReplayed);
395 LOG.info("-- Verifying edits from secondary");
396 verifyData(secondaryRegion, 0, 400, cq, families);
397
398 LOG.info("-- Verifying edits from primary. Ensuring that files are not deleted");
399 verifyData(primaryRegion, 0, lastReplayed, cq, families);
400 for (Store store : primaryRegion.getStores()) {
401 if (store.getColumnFamilyName().equals("cf1")) {
402 assertEquals(1, store.getStorefilesCount());
403 } else {
404 assertEquals(expectedStoreFileCount, store.getStorefilesCount());
405 }
406 }
407 }
408
409
410
411
412
413 @Test
414 public void testReplayFlushStartMarkers() throws IOException {
415
416 putDataWithFlushes(primaryRegion, 100, 100, 100);
417 int numRows = 200;
418
419
420 reader = createWALReaderForPrimary();
421
422 LOG.info("-- Replaying edits and flush events in secondary");
423
424 FlushDescriptor startFlushDesc = null;
425
426 int lastReplayed = 0;
427 while (true) {
428 WAL.Entry entry = reader.next();
429 if (entry == null) {
430 break;
431 }
432 FlushDescriptor flushDesc
433 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
434 if (flushDesc != null) {
435
436 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
437 long storeMemstoreSize = store.getMemStoreSize();
438 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
439 long storeFlushableSize = store.getFlushableSize();
440
441 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
442 startFlushDesc = flushDesc;
443 LOG.info("-- Replaying flush start in secondary");
444 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(startFlushDesc);
445 assertNull(result.result);
446 assertEquals(result.flushOpSeqId, startFlushDesc.getFlushSequenceNumber());
447 assertTrue(regionMemstoreSize > 0);
448 assertTrue(storeFlushableSize > 0);
449
450
451 long newStoreMemstoreSize = store.getMemStoreSize();
452 LOG.info("Memstore size reduced by:"
453 + StringUtils.humanReadableInt(newStoreMemstoreSize - storeMemstoreSize));
454 assertTrue(storeMemstoreSize > newStoreMemstoreSize);
455 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
456
457 }
458
459 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
460 } else {
461 lastReplayed = replayEdit(secondaryRegion, entry);
462 }
463 }
464
465
466
467
468 verifyData(secondaryRegion, 0, numRows, cq, families);
469
470
471 LOG.info("-- Replaying same flush start in secondary again");
472 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(startFlushDesc);
473 assertNull(result);
474
475 assertNotNull(secondaryRegion.getPrepareFlushResult());
476 assertEquals(secondaryRegion.getPrepareFlushResult().flushOpSeqId,
477 startFlushDesc.getFlushSequenceNumber());
478 assertTrue(secondaryRegion.getMemstoreSize() > 0);
479 verifyData(secondaryRegion, 0, numRows, cq, families);
480
481
482 FlushDescriptor startFlushDescSmallerSeqId
483 = clone(startFlushDesc, startFlushDesc.getFlushSequenceNumber() - 50);
484 LOG.info("-- Replaying same flush start in secondary again " + startFlushDescSmallerSeqId);
485 result = secondaryRegion.replayWALFlushStartMarker(startFlushDescSmallerSeqId);
486 assertNull(result);
487
488 assertNotNull(secondaryRegion.getPrepareFlushResult());
489 assertEquals(secondaryRegion.getPrepareFlushResult().flushOpSeqId,
490 startFlushDesc.getFlushSequenceNumber());
491 assertTrue(secondaryRegion.getMemstoreSize() > 0);
492 verifyData(secondaryRegion, 0, numRows, cq, families);
493
494
495 FlushDescriptor startFlushDescLargerSeqId
496 = clone(startFlushDesc, startFlushDesc.getFlushSequenceNumber() + 50);
497 LOG.info("-- Replaying same flush start in secondary again " + startFlushDescLargerSeqId);
498 result = secondaryRegion.replayWALFlushStartMarker(startFlushDescLargerSeqId);
499 assertNull(result);
500
501 assertNotNull(secondaryRegion.getPrepareFlushResult());
502 assertEquals(secondaryRegion.getPrepareFlushResult().flushOpSeqId,
503 startFlushDesc.getFlushSequenceNumber());
504 assertTrue(secondaryRegion.getMemstoreSize() > 0);
505 verifyData(secondaryRegion, 0, numRows, cq, families);
506
507 LOG.info("-- Verifying edits from secondary");
508 verifyData(secondaryRegion, 0, numRows, cq, families);
509
510 LOG.info("-- Verifying edits from primary.");
511 verifyData(primaryRegion, 0, numRows, cq, families);
512 }
513
514
515
516
517
518 @Test
519 public void testReplayFlushCommitMarkerSmallerThanFlushStartMarker() throws IOException {
520
521 putDataWithFlushes(primaryRegion, 100, 200, 100);
522 int numRows = 300;
523
524
525 reader = createWALReaderForPrimary();
526
527 LOG.info("-- Replaying edits and flush events in secondary");
528 FlushDescriptor startFlushDesc = null;
529 FlushDescriptor commitFlushDesc = null;
530
531 int lastReplayed = 0;
532 while (true) {
533 System.out.println(lastReplayed);
534 WAL.Entry entry = reader.next();
535 if (entry == null) {
536 break;
537 }
538 FlushDescriptor flushDesc
539 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
540 if (flushDesc != null) {
541 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
542
543 if (startFlushDesc == null) {
544 startFlushDesc = flushDesc;
545 } else {
546 LOG.info("-- Replaying flush start in secondary");
547 startFlushDesc = flushDesc;
548 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(startFlushDesc);
549 assertNull(result.result);
550 }
551 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
552
553 if (commitFlushDesc == null) {
554 commitFlushDesc = flushDesc;
555 }
556 }
557
558 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
559 } else {
560 lastReplayed = replayEdit(secondaryRegion, entry);
561 }
562 }
563
564
565
566 verifyData(secondaryRegion, 0, numRows, cq, families);
567
568
569 int expectedStoreFileCount = 0;
570 for (Store s : secondaryRegion.getStores()) {
571 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
572 }
573 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
574
575
576 LOG.info("Testing replaying flush COMMIT " + commitFlushDesc + " on top of flush START"
577 + startFlushDesc);
578 assertTrue(commitFlushDesc.getFlushSequenceNumber() < startFlushDesc.getFlushSequenceNumber());
579
580 LOG.info("-- Replaying flush commit in secondary" + commitFlushDesc);
581 secondaryRegion.replayWALFlushCommitMarker(commitFlushDesc);
582
583
584 expectedStoreFileCount++;
585 for (Store s : secondaryRegion.getStores()) {
586 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
587 }
588 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
589 long newFlushableSize = store.getFlushableSize();
590 assertTrue(newFlushableSize > 0);
591
592
593 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
594 assertEquals(regionMemstoreSize, newRegionMemstoreSize);
595
596 assertNotNull(secondaryRegion.getPrepareFlushResult());
597
598 LOG.info("-- Verifying edits from secondary");
599 verifyData(secondaryRegion, 0, numRows, cq, families);
600
601 LOG.info("-- Verifying edits from primary.");
602 verifyData(primaryRegion, 0, numRows, cq, families);
603 }
604
605
606
607
608
609 @Test
610 public void testReplayFlushCommitMarkerLargerThanFlushStartMarker() throws IOException {
611
612 putDataWithFlushes(primaryRegion, 100, 100, 100);
613 int numRows = 200;
614
615
616 reader = createWALReaderForPrimary();
617
618 LOG.info("-- Replaying edits and flush events in secondary");
619 FlushDescriptor startFlushDesc = null;
620 FlushDescriptor commitFlushDesc = null;
621
622 int lastReplayed = 0;
623 while (true) {
624 WAL.Entry entry = reader.next();
625 if (entry == null) {
626 break;
627 }
628 FlushDescriptor flushDesc
629 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
630 if (flushDesc != null) {
631 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
632 if (startFlushDesc == null) {
633 LOG.info("-- Replaying flush start in secondary");
634 startFlushDesc = flushDesc;
635 PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(startFlushDesc);
636 assertNull(result.result);
637 }
638 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
639
640
641
642 commitFlushDesc =
643 FlushDescriptor.newBuilder(flushDesc)
644 .setFlushSequenceNumber(flushDesc.getFlushSequenceNumber() + 50)
645 .build();
646 }
647
648 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
649 } else {
650 lastReplayed = replayEdit(secondaryRegion, entry);
651 }
652 }
653
654
655
656 verifyData(secondaryRegion, 0, numRows, cq, families);
657
658
659 int expectedStoreFileCount = 0;
660 for (Store s : secondaryRegion.getStores()) {
661 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
662 }
663 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
664
665
666 LOG.info("Testing replaying flush COMMIT " + commitFlushDesc + " on top of flush START"
667 + startFlushDesc);
668 assertTrue(commitFlushDesc.getFlushSequenceNumber() > startFlushDesc.getFlushSequenceNumber());
669
670 LOG.info("-- Replaying flush commit in secondary" + commitFlushDesc);
671 secondaryRegion.replayWALFlushCommitMarker(commitFlushDesc);
672
673
674 expectedStoreFileCount++;
675 for (Store s : secondaryRegion.getStores()) {
676 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
677 }
678 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
679 long newFlushableSize = store.getFlushableSize();
680 assertTrue(newFlushableSize > 0);
681
682
683 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
684 assertTrue(newRegionMemstoreSize > 0);
685 assertTrue(regionMemstoreSize > newRegionMemstoreSize);
686
687 assertNull(secondaryRegion.getPrepareFlushResult());
688
689 LOG.info("-- Verifying edits from secondary");
690 verifyData(secondaryRegion, 0, numRows, cq, families);
691
692 LOG.info("-- Verifying edits from primary.");
693 verifyData(primaryRegion, 0, numRows, cq, families);
694 }
695
696
697
698
699
700
701 @Test
702 public void testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore()
703 throws IOException {
704 testReplayFlushCommitMarkerWithoutFlushStartMarker(true);
705 }
706
707
708
709
710
711
712 @Test
713 public void testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore()
714 throws IOException {
715 testReplayFlushCommitMarkerWithoutFlushStartMarker(false);
716 }
717
718
719
720
721 public void testReplayFlushCommitMarkerWithoutFlushStartMarker(boolean droppableMemstore)
722 throws IOException {
723
724
725 putDataWithFlushes(primaryRegion, 100, 100, droppableMemstore ? 0 : 100);
726 int numRows = droppableMemstore ? 100 : 200;
727
728
729 reader = createWALReaderForPrimary();
730
731 LOG.info("-- Replaying edits and flush events in secondary");
732 FlushDescriptor commitFlushDesc = null;
733
734 int lastReplayed = 0;
735 while (true) {
736 WAL.Entry entry = reader.next();
737 if (entry == null) {
738 break;
739 }
740 FlushDescriptor flushDesc
741 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
742 if (flushDesc != null) {
743 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
744
745 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
746 commitFlushDesc = flushDesc;
747 }
748
749 verifyData(secondaryRegion, 0, lastReplayed+1, cq, families);
750 } else {
751 lastReplayed = replayEdit(secondaryRegion, entry);
752 }
753 }
754
755
756
757 verifyData(secondaryRegion, 0, numRows, cq, families);
758
759
760 int expectedStoreFileCount = 0;
761 for (Store s : secondaryRegion.getStores()) {
762 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
763 }
764 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
765
766
767 assertNull(secondaryRegion.getPrepareFlushResult());
768 assertTrue(commitFlushDesc.getFlushSequenceNumber() > 0);
769
770
771 for (Store store : secondaryRegion.getStores()) {
772 assertTrue(store.getMaxSequenceId() <= secondaryRegion.getSequenceId());
773 }
774
775 LOG.info("-- Replaying flush commit in secondary" + commitFlushDesc);
776 secondaryRegion.replayWALFlushCommitMarker(commitFlushDesc);
777
778
779 expectedStoreFileCount++;
780 for (Store s : secondaryRegion.getStores()) {
781 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
782 }
783 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
784 long newFlushableSize = store.getFlushableSize();
785 if (droppableMemstore) {
786 assertTrue(newFlushableSize == 0);
787 } else {
788 assertTrue(newFlushableSize > 0);
789 }
790
791
792 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
793 if (droppableMemstore) {
794 assertTrue(0 == newRegionMemstoreSize);
795 } else {
796 assertTrue(regionMemstoreSize == newRegionMemstoreSize);
797 }
798
799 LOG.info("-- Verifying edits from secondary");
800 verifyData(secondaryRegion, 0, numRows, cq, families);
801
802 LOG.info("-- Verifying edits from primary.");
803 verifyData(primaryRegion, 0, numRows, cq, families);
804 }
805
806 private FlushDescriptor clone(FlushDescriptor flush, long flushSeqId) {
807 return FlushDescriptor.newBuilder(flush)
808 .setFlushSequenceNumber(flushSeqId)
809 .build();
810 }
811
812
813
814
815 @Test
816 public void testReplayRegionOpenEvent() throws IOException {
817 putDataWithFlushes(primaryRegion, 100, 0, 100);
818 int numRows = 100;
819
820
821 primaryRegion.close();
822 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
823
824
825 reader = createWALReaderForPrimary();
826 List<RegionEventDescriptor> regionEvents = Lists.newArrayList();
827
828 LOG.info("-- Replaying edits and region events in secondary");
829 while (true) {
830 WAL.Entry entry = reader.next();
831 if (entry == null) {
832 break;
833 }
834 FlushDescriptor flushDesc
835 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
836 RegionEventDescriptor regionEventDesc
837 = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
838
839 if (flushDesc != null) {
840
841 } else if (regionEventDesc != null) {
842 regionEvents.add(regionEventDesc);
843 } else {
844
845 }
846 }
847
848
849 assertEquals(3, regionEvents.size());
850
851
852 secondaryRegion.replayWALRegionEventMarker(regionEvents.get(0));
853
854
855 secondaryRegion.replayWALRegionEventMarker(regionEvents.get(1));
856
857
858 int expectedStoreFileCount = 0;
859 for (Store s : secondaryRegion.getStores()) {
860 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
861 }
862 long regionMemstoreSize = secondaryRegion.getMemstoreSize();
863 assertTrue(regionMemstoreSize == 0);
864
865
866 LOG.info("Testing replaying region open event " + regionEvents.get(2));
867 secondaryRegion.replayWALRegionEventMarker(regionEvents.get(2));
868
869
870 expectedStoreFileCount++;
871 for (Store s : secondaryRegion.getStores()) {
872 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
873 }
874 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
875 long newFlushableSize = store.getFlushableSize();
876 assertTrue(newFlushableSize == 0);
877
878
879 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
880 assertTrue(newRegionMemstoreSize == 0);
881
882 assertNull(secondaryRegion.getPrepareFlushResult());
883
884 LOG.info("-- Verifying edits from secondary");
885 verifyData(secondaryRegion, 0, numRows, cq, families);
886
887 LOG.info("-- Verifying edits from primary.");
888 verifyData(primaryRegion, 0, numRows, cq, families);
889 }
890
891
892
893
894
895 @Test
896 public void testReplayRegionOpenEventAfterFlushStart() throws IOException {
897 putDataWithFlushes(primaryRegion, 100, 100, 100);
898 int numRows = 200;
899
900
901 primaryRegion.close();
902 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
903
904
905 reader = createWALReaderForPrimary();
906 List<RegionEventDescriptor> regionEvents = Lists.newArrayList();
907
908 LOG.info("-- Replaying edits and region events in secondary");
909 while (true) {
910 WAL.Entry entry = reader.next();
911 if (entry == null) {
912 break;
913 }
914 FlushDescriptor flushDesc
915 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
916 RegionEventDescriptor regionEventDesc
917 = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
918
919 if (flushDesc != null) {
920
921 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
922 secondaryRegion.replayWALFlushStartMarker(flushDesc);
923 }
924 } else if (regionEventDesc != null) {
925 regionEvents.add(regionEventDesc);
926 } else {
927 replayEdit(secondaryRegion, entry);
928 }
929 }
930
931
932
933 verifyData(secondaryRegion, 0, numRows, cq, families);
934
935
936 assertEquals(3, regionEvents.size());
937
938
939 int expectedStoreFileCount = 0;
940 for (Store s : secondaryRegion.getStores()) {
941 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
942 }
943
944
945 LOG.info("Testing replaying region open event " + regionEvents.get(2));
946 secondaryRegion.replayWALRegionEventMarker(regionEvents.get(2));
947
948
949 expectedStoreFileCount = 2;
950 for (Store s : secondaryRegion.getStores()) {
951 assertEquals(expectedStoreFileCount, s.getStorefilesCount());
952 }
953 Store store = secondaryRegion.getStore(Bytes.toBytes("cf1"));
954 long newSnapshotSize = store.getSnapshotSize();
955 assertTrue(newSnapshotSize == 0);
956
957
958 long newRegionMemstoreSize = secondaryRegion.getMemstoreSize();
959 assertTrue(newRegionMemstoreSize == 0);
960
961 assertNull(secondaryRegion.getPrepareFlushResult());
962
963 LOG.info("-- Verifying edits from secondary");
964 verifyData(secondaryRegion, 0, numRows, cq, families);
965
966 LOG.info("-- Verifying edits from primary.");
967 verifyData(primaryRegion, 0, numRows, cq, families);
968 }
969
970
971
972
973
974 @Test
975 public void testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent() throws IOException {
976 putDataWithFlushes(primaryRegion, 100, 100, 0);
977 int numRows = 100;
978
979
980 primaryRegion.close();
981 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
982
983
984 reader = createWALReaderForPrimary();
985 List<RegionEventDescriptor> regionEvents = Lists.newArrayList();
986 List<WAL.Entry> edits = Lists.newArrayList();
987
988 LOG.info("-- Replaying edits and region events in secondary");
989 while (true) {
990 WAL.Entry entry = reader.next();
991 if (entry == null) {
992 break;
993 }
994 FlushDescriptor flushDesc
995 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
996 RegionEventDescriptor regionEventDesc
997 = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
998
999 if (flushDesc != null) {
1000
1001 } else if (regionEventDesc != null) {
1002 regionEvents.add(regionEventDesc);
1003 } else {
1004 edits.add(entry);
1005 }
1006 }
1007
1008
1009
1010 secondaryRegion.replayWALRegionEventMarker(
1011 RegionEventDescriptor.newBuilder(regionEvents.get(0)).setLogSequenceNumber(
1012 regionEvents.get(2).getLogSequenceNumber()).build());
1013
1014
1015
1016
1017 for (WAL.Entry entry: edits) {
1018 replayEdit(secondaryRegion, entry);
1019 }
1020
1021 boolean expectedFail = false;
1022 try {
1023 verifyData(secondaryRegion, 0, numRows, cq, families);
1024 } catch (AssertionError e) {
1025 expectedFail = true;
1026 }
1027 if (!expectedFail) {
1028 fail("Should have failed this verification");
1029 }
1030 }
1031
1032 @Test
1033 public void testReplayFlushSeqIds() throws IOException {
1034
1035 int start = 0;
1036 LOG.info("-- Writing some data to primary from " + start + " to " + (start+100));
1037 putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
1038 LOG.info("-- Flushing primary, creating 3 files for 3 stores");
1039 primaryRegion.flush(true);
1040
1041
1042 reader = createWALReaderForPrimary();
1043
1044 long flushSeqId = -1;
1045 LOG.info("-- Replaying flush events in secondary");
1046 while (true) {
1047 WAL.Entry entry = reader.next();
1048 if (entry == null) {
1049 break;
1050 }
1051 FlushDescriptor flushDesc
1052 = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1053 if (flushDesc != null) {
1054 if (flushDesc.getAction() == FlushAction.START_FLUSH) {
1055 LOG.info("-- Replaying flush start in secondary");
1056 secondaryRegion.replayWALFlushStartMarker(flushDesc);
1057 flushSeqId = flushDesc.getFlushSequenceNumber();
1058 } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
1059 LOG.info("-- Replaying flush commit in secondary");
1060 secondaryRegion.replayWALFlushCommitMarker(flushDesc);
1061 assertEquals(flushSeqId, flushDesc.getFlushSequenceNumber());
1062 }
1063 }
1064
1065 }
1066
1067
1068
1069 long readPoint = secondaryRegion.getMVCC().getReadPoint();
1070 assertEquals(flushSeqId, readPoint);
1071
1072
1073 verifyData(secondaryRegion, 0, 100, cq, families);
1074 }
1075
1076 @Test
1077 public void testSeqIdsFromReplay() throws IOException {
1078
1079
1080 String method = name.getMethodName();
1081 byte[] tableName = Bytes.toBytes(method);
1082 byte[] family = Bytes.toBytes("family");
1083
1084 HRegion region = initHRegion(tableName, method, family);
1085 try {
1086
1087 long readPoint = region.getMVCC().getReadPoint();
1088 long origSeqId = readPoint + 100;
1089
1090 Put put = new Put(row).add(family, row, row);
1091 put.setDurability(Durability.SKIP_WAL);
1092 replay(region, put, origSeqId);
1093
1094
1095 assertGet(region, family, row);
1096
1097
1098 assertEquals(origSeqId, region.getSequenceId());
1099
1100
1101
1102
1103 put = new Put(row2).add(family, row2, row2);
1104 put.setDurability(Durability.SKIP_WAL);
1105 replay(region, put, origSeqId - 50);
1106
1107 assertGet(region, family, row2);
1108 } finally {
1109 region.close();
1110 }
1111 }
1112
1113
1114
1115
1116
1117
1118 @SuppressWarnings("unchecked")
1119 @Test
1120 public void testSecondaryRegionDoesNotWriteRegionEventsToWAL() throws IOException {
1121 secondaryRegion.close();
1122 walSecondary = spy(walSecondary);
1123
1124
1125 secondaryRegion = HRegion.openHRegion(secondaryHri, htd, walSecondary, CONF, rss, null);
1126 verify(walSecondary, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(),
1127 (WALKey)any(), (WALEdit)any(), anyBoolean());
1128
1129
1130 putDataByReplay(secondaryRegion, 0, 10, cq, families);
1131 secondaryRegion.replayWALFlushStartMarker(FlushDescriptor.newBuilder().
1132 setFlushSequenceNumber(10)
1133 .setTableName(ByteString.copyFrom(primaryRegion.getTableDesc().getTableName().getName()))
1134 .setAction(FlushAction.START_FLUSH)
1135 .setEncodedRegionName(
1136 ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes()))
1137 .setRegionName(ByteString.copyFrom(primaryRegion.getRegionInfo().getRegionName()))
1138 .build());
1139
1140 verify(walSecondary, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(),
1141 (WALKey)any(), (WALEdit)any(), anyBoolean());
1142
1143 secondaryRegion.close();
1144 verify(walSecondary, times(0)).append((HTableDescriptor)any(), (HRegionInfo)any(),
1145 (WALKey)any(), (WALEdit)any(), anyBoolean());
1146 }
1147
1148
1149
1150
1151 @Test
1152 public void testRegionReadsEnabledFlag() throws IOException {
1153
1154 putDataByReplay(secondaryRegion, 0, 100, cq, families);
1155
1156 verifyData(secondaryRegion, 0, 100, cq, families);
1157
1158
1159 secondaryRegion.setReadsEnabled(false);
1160 try {
1161 verifyData(secondaryRegion, 0, 100, cq, families);
1162 fail("Should have failed with IOException");
1163 } catch(IOException ex) {
1164
1165 }
1166
1167
1168 putDataByReplay(secondaryRegion, 100, 100, cq, families);
1169
1170
1171 secondaryRegion.setReadsEnabled(true);
1172 verifyData(secondaryRegion, 0, 200, cq, families);
1173 }
1174
1175
1176
1177
1178
1179 @Test
1180 public void testWriteFlushRequestMarker() throws IOException {
1181
1182 FlushResultImpl result = (FlushResultImpl)((HRegion)primaryRegion).flushcache(true, false);
1183 assertNotNull(result);
1184 assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
1185 assertFalse(result.wroteFlushWalMarker);
1186
1187
1188 result = (FlushResultImpl)((HRegion)primaryRegion).flushcache(true, true);
1189 assertNotNull(result);
1190 assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
1191 assertTrue(result.wroteFlushWalMarker);
1192
1193 List<FlushDescriptor> flushes = Lists.newArrayList();
1194 reader = createWALReaderForPrimary();
1195 while (true) {
1196 WAL.Entry entry = reader.next();
1197 if (entry == null) {
1198 break;
1199 }
1200 FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1201 if (flush != null) {
1202 flushes.add(flush);
1203 }
1204 }
1205
1206 assertEquals(1, flushes.size());
1207 assertNotNull(flushes.get(0));
1208 assertEquals(FlushDescriptor.FlushAction.CANNOT_FLUSH, flushes.get(0).getAction());
1209 }
1210
1211
1212
1213
1214
1215
1216
1217 @Test
1218 public void testReplayingFlushRequestRestoresReadsEnabledState() throws IOException {
1219 disableReads(secondaryRegion);
1220
1221
1222
1223 primaryRegion.flushcache(true, true);
1224 reader = createWALReaderForPrimary();
1225 while (true) {
1226 WAL.Entry entry = reader.next();
1227 if (entry == null) {
1228 break;
1229 }
1230 FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1231 if (flush != null) {
1232 secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum());
1233 }
1234 }
1235
1236
1237 secondaryRegion.get(new Get(Bytes.toBytes(0)));
1238 }
1239
1240
1241
1242
1243
1244
1245
1246 @Test
1247 public void testReplayingFlushRestoresReadsEnabledState() throws IOException {
1248
1249
1250 disableReads(secondaryRegion);
1251
1252
1253 putData(primaryRegion, Durability.SYNC_WAL, 0, 100, cq, families);
1254 primaryRegion.flush(true);
1255
1256 reader = createWALReaderForPrimary();
1257 while (true) {
1258 WAL.Entry entry = reader.next();
1259 if (entry == null) {
1260 break;
1261 }
1262 FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1263 if (flush != null) {
1264 secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum());
1265 } else {
1266 replayEdit(secondaryRegion, entry);
1267 }
1268 }
1269
1270
1271 verifyData(secondaryRegion, 0, 100, cq, families);
1272 }
1273
1274
1275
1276
1277
1278
1279
1280 @Test
1281 public void testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState() throws IOException {
1282
1283
1284 disableReads(secondaryRegion);
1285
1286
1287 putData(primaryRegion, Durability.SYNC_WAL, 0, 100, cq, families);
1288 primaryRegion.flush(true);
1289
1290 reader = createWALReaderForPrimary();
1291 while (true) {
1292 WAL.Entry entry = reader.next();
1293 if (entry == null) {
1294 break;
1295 }
1296 FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1297 if (flush != null) {
1298 secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum());
1299 }
1300 }
1301
1302
1303 verifyData(secondaryRegion, 0, 100, cq, families);
1304 }
1305
1306
1307
1308
1309
1310
1311
1312 @Test
1313 public void testReplayingRegionOpenEventRestoresReadsEnabledState() throws IOException {
1314
1315 disableReads(secondaryRegion);
1316
1317 primaryRegion.close();
1318 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
1319
1320 reader = createWALReaderForPrimary();
1321 while (true) {
1322 WAL.Entry entry = reader.next();
1323 if (entry == null) {
1324 break;
1325 }
1326
1327 RegionEventDescriptor regionEventDesc
1328 = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));
1329
1330 if (regionEventDesc != null) {
1331 secondaryRegion.replayWALRegionEventMarker(regionEventDesc);
1332 }
1333 }
1334
1335
1336 secondaryRegion.get(new Get(Bytes.toBytes(0)));
1337 }
1338
1339 @Test
1340 public void testRefreshStoreFiles() throws IOException {
1341 assertEquals(0, primaryRegion.getStoreFileList(families).size());
1342 assertEquals(0, secondaryRegion.getStoreFileList(families).size());
1343
1344
1345 secondaryRegion.refreshStoreFiles();
1346 assertEquals(0, secondaryRegion.getStoreFileList(families).size());
1347
1348
1349 putDataWithFlushes(primaryRegion, 100, 100, 0);
1350 int numRows = 100;
1351
1352
1353 secondaryRegion.refreshStoreFiles();
1354 assertPathListsEqual(primaryRegion.getStoreFileList(families),
1355 secondaryRegion.getStoreFileList(families));
1356 assertEquals(families.length, secondaryRegion.getStoreFileList(families).size());
1357
1358 LOG.info("-- Verifying edits from secondary");
1359 verifyData(secondaryRegion, 0, numRows, cq, families);
1360
1361
1362 putDataWithFlushes(primaryRegion, 100, 300, 0);
1363 numRows = 300;
1364
1365
1366 secondaryRegion.refreshStoreFiles();
1367 assertPathListsEqual(primaryRegion.getStoreFileList(families),
1368 secondaryRegion.getStoreFileList(families));
1369 assertEquals(families.length * 4, secondaryRegion.getStoreFileList(families).size());
1370
1371 LOG.info("-- Verifying edits from secondary");
1372 verifyData(secondaryRegion, 0, numRows, cq, families);
1373
1374 if (FSUtils.WINDOWS) {
1375
1376 return;
1377 }
1378
1379
1380 primaryRegion.compactStores();
1381 List<Region> regions = new ArrayList<Region>();
1382 regions.add(primaryRegion);
1383 when(rss.getOnlineRegions()).thenReturn(regions);
1384 CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, rss, false);
1385 cleaner.chore();
1386 secondaryRegion.refreshStoreFiles();
1387 assertPathListsEqual(primaryRegion.getStoreFileList(families),
1388 secondaryRegion.getStoreFileList(families));
1389 assertEquals(families.length, secondaryRegion.getStoreFileList(families).size());
1390
1391 LOG.info("-- Verifying edits from secondary");
1392 verifyData(secondaryRegion, 0, numRows, cq, families);
1393
1394 LOG.info("-- Replaying edits in secondary");
1395
1396
1397 assertTrue(secondaryRegion.getMemstoreSize() == 0);
1398 putDataWithFlushes(primaryRegion, 400, 400, 0);
1399 numRows = 400;
1400
1401 reader = createWALReaderForPrimary();
1402 while (true) {
1403 WAL.Entry entry = reader.next();
1404 if (entry == null) {
1405 break;
1406 }
1407 FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
1408 if (flush != null) {
1409
1410 } else {
1411 replayEdit(secondaryRegion, entry);
1412 }
1413 }
1414
1415 assertTrue(secondaryRegion.getMemstoreSize() > 0);
1416
1417 secondaryRegion.refreshStoreFiles();
1418
1419 assertTrue(secondaryRegion.getMemstoreSize() == 0);
1420
1421 LOG.info("-- Verifying edits from primary");
1422 verifyData(primaryRegion, 0, numRows, cq, families);
1423 LOG.info("-- Verifying edits from secondary");
1424 verifyData(secondaryRegion, 0, numRows, cq, families);
1425 }
1426
1427
1428
1429
1430 private void assertPathListsEqual(List<String> list1, List<String> list2) {
1431 List<Path> l1 = new ArrayList<>(list1.size());
1432 for (String path : list1) {
1433 l1.add(Path.getPathWithoutSchemeAndAuthority(new Path(path)));
1434 }
1435 List<Path> l2 = new ArrayList<>(list2.size());
1436 for (String path : list2) {
1437 l2.add(Path.getPathWithoutSchemeAndAuthority(new Path(path)));
1438 }
1439 assertEquals(l1, l2);
1440 }
1441
1442 private void disableReads(HRegion region) {
1443 region.setReadsEnabled(false);
1444 try {
1445 verifyData(region, 0, 1, cq, families);
1446 fail("Should have failed with IOException");
1447 } catch(IOException ex) {
1448
1449 }
1450 }
1451
1452 private void replay(HRegion region, Put put, long replaySeqId) throws IOException {
1453 put.setDurability(Durability.SKIP_WAL);
1454 MutationReplay mutation = new MutationReplay(MutationType.PUT, put, 0, 0);
1455 region.batchReplay(new MutationReplay[] {mutation}, replaySeqId);
1456 }
1457
1458
1459
1460
1461 @Test
1462 public void testReplayBulkLoadEvent() throws IOException {
1463 LOG.info("testReplayBulkLoadEvent starts");
1464 putDataWithFlushes(primaryRegion, 100, 0, 100);
1465
1466
1467 primaryRegion.close();
1468 primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
1469
1470
1471 Random random = new Random();
1472 byte[] randomValues = new byte[20];
1473 random.nextBytes(randomValues);
1474 Path testPath = TEST_UTIL.getDataTestDirOnTestFS();
1475
1476 List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
1477 int expectedLoadFileCount = 0;
1478 for (byte[] family : families) {
1479 familyPaths.add(new Pair<byte[], String>(family, createHFileForFamilies(testPath, family,
1480 randomValues)));
1481 expectedLoadFileCount++;
1482 }
1483 primaryRegion.bulkLoadHFiles(familyPaths, false, null, null);
1484
1485
1486 reader = createWALReaderForPrimary();
1487
1488 LOG.info("-- Replaying edits and region events in secondary");
1489 BulkLoadDescriptor bulkloadEvent = null;
1490 while (true) {
1491 WAL.Entry entry = reader.next();
1492 if (entry == null) {
1493 break;
1494 }
1495 bulkloadEvent = WALEdit.getBulkLoadDescriptor(entry.getEdit().getCells().get(0));
1496 if (bulkloadEvent != null) {
1497 break;
1498 }
1499 }
1500
1501
1502 assertTrue(bulkloadEvent != null);
1503 assertEquals(expectedLoadFileCount, bulkloadEvent.getStoresCount());
1504
1505
1506 secondaryRegion.replayWALBulkLoadEventMarker(bulkloadEvent);
1507
1508
1509 List<String> storeFileName = new ArrayList<String>();
1510 for (StoreDescriptor storeDesc : bulkloadEvent.getStoresList()) {
1511 storeFileName.addAll(storeDesc.getStoreFileList());
1512 }
1513
1514 for (Store s : secondaryRegion.getStores()) {
1515 for (StoreFile sf : s.getStorefiles()) {
1516 storeFileName.remove(sf.getPath().getName());
1517 }
1518 }
1519 assertTrue("Found some store file isn't loaded:" + storeFileName, storeFileName.isEmpty());
1520
1521 LOG.info("-- Verifying edits from secondary");
1522 for (byte[] family : families) {
1523 assertGet(secondaryRegion, family, randomValues);
1524 }
1525 }
1526
1527 @Test
1528 public void testReplayingFlushCommitWithFileAlreadyDeleted() throws IOException {
1529
1530
1531 secondaryRegion.replayWALFlushCommitMarker(FlushDescriptor.newBuilder().
1532 setFlushSequenceNumber(Long.MAX_VALUE)
1533 .setTableName(ByteString.copyFrom(primaryRegion.getTableDesc().getTableName().getName()))
1534 .setAction(FlushAction.COMMIT_FLUSH)
1535 .setEncodedRegionName(
1536 ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes()))
1537 .setRegionName(ByteString.copyFrom(primaryRegion.getRegionInfo().getRegionName()))
1538 .addStoreFlushes(StoreFlushDescriptor.newBuilder()
1539 .setFamilyName(ByteString.copyFrom(families[0]))
1540 .setStoreHomeDir("/store_home_dir")
1541 .addFlushOutput("/foo/baz/123")
1542 .build())
1543 .build());
1544 }
1545
1546 @Test
1547 public void testReplayingCompactionWithFileAlreadyDeleted() throws IOException {
1548
1549
1550 secondaryRegion.replayWALCompactionMarker(CompactionDescriptor.newBuilder()
1551 .setTableName(ByteString.copyFrom(primaryRegion.getTableDesc().getTableName().getName()))
1552 .setEncodedRegionName(
1553 ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes()))
1554 .setFamilyName(ByteString.copyFrom(families[0]))
1555 .addCompactionInput("/123")
1556 .addCompactionOutput("/456")
1557 .setStoreHomeDir("/store_home_dir")
1558 .setRegionName(ByteString.copyFrom(primaryRegion.getRegionInfo().getRegionName()))
1559 .build()
1560 , true, true, Long.MAX_VALUE);
1561 }
1562
1563 @Test
1564 public void testReplayingRegionOpenEventWithFileAlreadyDeleted() throws IOException {
1565
1566
1567 secondaryRegion.replayWALRegionEventMarker(RegionEventDescriptor.newBuilder()
1568 .setTableName(ByteString.copyFrom(primaryRegion.getTableDesc().getTableName().getName()))
1569 .setEncodedRegionName(
1570 ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes()))
1571 .setRegionName(ByteString.copyFrom(primaryRegion.getRegionInfo().getRegionName()))
1572 .setEventType(EventType.REGION_OPEN)
1573 .setServer(ProtobufUtil.toServerName(ServerName.valueOf("foo", 1, 1)))
1574 .setLogSequenceNumber(Long.MAX_VALUE)
1575 .addStores(StoreDescriptor.newBuilder()
1576 .setFamilyName(ByteString.copyFrom(families[0]))
1577 .setStoreHomeDir("/store_home_dir")
1578 .addStoreFile("/123")
1579 .build())
1580 .build());
1581 }
1582
1583 @Test
1584 public void testReplayingBulkLoadEventWithFileAlreadyDeleted() throws IOException {
1585
1586
1587 secondaryRegion.replayWALBulkLoadEventMarker(BulkLoadDescriptor.newBuilder()
1588 .setTableName(ProtobufUtil.toProtoTableName(primaryRegion.getTableDesc().getTableName()))
1589 .setEncodedRegionName(
1590 ByteString.copyFrom(primaryRegion.getRegionInfo().getEncodedNameAsBytes()))
1591 .setBulkloadSeqNum(Long.MAX_VALUE)
1592 .addStores(StoreDescriptor.newBuilder()
1593 .setFamilyName(ByteString.copyFrom(families[0]))
1594 .setStoreHomeDir("/store_home_dir")
1595 .addStoreFile("/123")
1596 .build())
1597 .build());
1598 }
1599
1600 private String createHFileForFamilies(Path testPath, byte[] family,
1601 byte[] valueBytes) throws IOException {
1602 HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
1603
1604 Path testFile = new Path(testPath, UUID.randomUUID().toString());
1605 FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
1606 try {
1607 hFileFactory.withOutputStream(out);
1608 hFileFactory.withFileContext(new HFileContext());
1609 HFile.Writer writer = hFileFactory.create();
1610 try {
1611 writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0l,
1612 KeyValue.Type.Put.getCode(), valueBytes)));
1613 } finally {
1614 writer.close();
1615 }
1616 } finally {
1617 out.close();
1618 }
1619 return testFile.toString();
1620 }
1621
1622
1623
1624
1625
1626 private void putDataWithFlushes(HRegion region, int flushInterval,
1627 int numRows, int numRowsAfterFlush) throws IOException {
1628 int start = 0;
1629 for (; start < numRows; start += flushInterval) {
1630 LOG.info("-- Writing some data to primary from " + start + " to " + (start+flushInterval));
1631 putData(region, Durability.SYNC_WAL, start, flushInterval, cq, families);
1632 LOG.info("-- Flushing primary, creating 3 files for 3 stores");
1633 region.flush(true);
1634 }
1635 LOG.info("-- Writing some more data to primary, not flushing");
1636 putData(region, Durability.SYNC_WAL, start, numRowsAfterFlush, cq, families);
1637 }
1638
1639 private void putDataByReplay(HRegion region,
1640 int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
1641 for (int i = startRow; i < startRow + numRows; i++) {
1642 Put put = new Put(Bytes.toBytes("" + i));
1643 put.setDurability(Durability.SKIP_WAL);
1644 for (byte[] family : families) {
1645 put.add(family, qf, EnvironmentEdgeManager.currentTime(), null);
1646 }
1647 replay(region, put, i+1);
1648 }
1649 }
1650
1651 private static HRegion initHRegion(byte[] tableName,
1652 String callingMethod, byte[]... families) throws IOException {
1653 return initHRegion(tableName, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
1654 callingMethod, TEST_UTIL.getConfiguration(), false, Durability.SYNC_WAL, null, families);
1655 }
1656
1657 private static HRegion initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey,
1658 String callingMethod, Configuration conf, boolean isReadOnly, Durability durability,
1659 WAL wal, byte[]... families) throws IOException {
1660 return TEST_UTIL.createLocalHRegion(tableName, startKey, stopKey, callingMethod, conf,
1661 isReadOnly, durability, wal, families);
1662 }
1663 }