1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FileSystem;
24 import org.apache.hadoop.fs.Path;
25 import org.apache.hadoop.hbase.HBaseTestingUtility;
26 import org.apache.hadoop.hbase.HTableDescriptor;
27 import org.apache.hadoop.hbase.TableName;
28 import org.apache.hadoop.hbase.client.Table;
29 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
30 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
31 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos;
32 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
33 import org.apache.hadoop.hbase.regionserver.HRegion;
34 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
35 import org.apache.hadoop.hbase.regionserver.snapshot.FlushSnapshotSubprocedure;
36 import org.apache.hadoop.hbase.testclassification.MediumTests;
37 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.FSUtils;
40
41 import org.junit.AfterClass;
42 import org.junit.BeforeClass;
43 import org.junit.Test;
44 import org.junit.experimental.categories.Category;
45 import org.mockito.invocation.InvocationOnMock;
46 import org.mockito.stubbing.Answer;
47
48 import java.io.IOException;
49 import java.util.List;
50 import java.util.concurrent.ExecutorService;
51 import java.util.concurrent.Executors;
52 import java.util.concurrent.Future;
53
54 import static org.junit.Assert.assertEquals;
55 import static org.junit.Assert.assertNotNull;
56 import static org.junit.Assert.assertTrue;
57 import static org.mockito.Mockito.doAnswer;
58 import static org.mockito.Mockito.spy;
59
60
61
62
63
64 @Category({ MediumTests.class, RegionServerTests.class})
65 public class TestRegionSnapshotTask {
66 private final Log LOG = LogFactory.getLog(getClass());
67
68 private static HBaseTestingUtility TEST_UTIL;
69 private static Configuration conf;
70 private static FileSystem fs;
71 private static Path rootDir;
72
73 @BeforeClass
74 public static void setupBeforeClass() throws Exception {
75 TEST_UTIL = new HBaseTestingUtility();
76
77 conf = TEST_UTIL.getConfiguration();
78
79
80 conf.setInt("hbase.hfile.compaction.discharger.interval", 1000);
81 conf.setInt("hbase.master.hfilecleaner.ttl", 1000);
82
83 TEST_UTIL.startMiniCluster(1);
84 TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster();
85 TEST_UTIL.waitUntilAllRegionsAssigned(TableName.META_TABLE_NAME);
86
87 rootDir = FSUtils.getRootDir(conf);
88 fs = TEST_UTIL.getTestFileSystem();
89 }
90
91 @AfterClass
92 public static void tearDown() throws Exception {
93 TEST_UTIL.shutdownMiniCluster();
94 }
95
96
97
98
99
100
101
102
103
104
105 @Test(timeout = 30000)
106 public void testAddRegionWithCompactions() throws Exception {
107 final TableName tableName = TableName.valueOf("test_table");
108 Table table = setupTable(tableName);
109
110 List<HRegion> hRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
111
112 final HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder()
113 .setTable(tableName.getNameAsString())
114 .setType(HBaseProtos.SnapshotDescription.Type.FLUSH)
115 .setName("test_table_snapshot")
116 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
117 .build();
118 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(snapshot.getName());
119
120 final HRegion region = spy(hRegions.get(0));
121
122 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir, conf);
123 final SnapshotManifest manifest =
124 SnapshotManifest.create(conf, fs, workingDir, snapshot, monitor);
125 manifest.addTableDescriptor(table.getTableDescriptor());
126
127 if (!fs.exists(workingDir)) {
128 fs.mkdirs(workingDir);
129 }
130 assertTrue(fs.exists(workingDir));
131 SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
132
133 doAnswer(new Answer<Void>() {
134 @Override
135 public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
136 addRegionToSnapshot(snapshot, region, manifest);
137 return null;
138 }
139 }).when(region).addRegionToSnapshot(snapshot, monitor);
140
141 FlushSnapshotSubprocedure.RegionSnapshotTask snapshotTask =
142 new FlushSnapshotSubprocedure.RegionSnapshotTask(region, snapshot, true, monitor);
143 ExecutorService executor = Executors.newFixedThreadPool(1);
144 Future f = executor.submit(snapshotTask);
145
146
147 LOG.info("Starting major compaction");
148 region.compact(true);
149 LOG.info("Finished major compaction");
150 f.get();
151
152
153 manifest.consolidate();
154
155
156 assertNotNull(manifest.getRegionManifests());
157
158 assertEquals(1, manifest.getRegionManifests().size());
159
160
161 SnapshotReferenceUtil.verifySnapshot(conf, fs, manifest);
162 }
163
164 private void addRegionToSnapshot(HBaseProtos.SnapshotDescription snapshot,
165 HRegion region, SnapshotManifest manifest) throws Exception {
166 LOG.info("Adding region to snapshot: " + region.getRegionInfo().getRegionNameAsString());
167 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir, conf);
168 SnapshotManifest.RegionVisitor visitor = createRegionVisitorWithDelay(snapshot, workingDir);
169 manifest.addRegion(region, visitor);
170 LOG.info("Added the region to snapshot: " + region.getRegionInfo().getRegionNameAsString());
171 }
172
173 private SnapshotManifest.RegionVisitor createRegionVisitorWithDelay(
174 HBaseProtos.SnapshotDescription desc, Path workingDir) {
175 return new SnapshotManifestV2.ManifestBuilder(conf, fs, workingDir) {
176 @Override
177 public void storeFile(final SnapshotProtos.SnapshotRegionManifest.Builder region,
178 final SnapshotProtos.SnapshotRegionManifest.FamilyFiles.Builder family,
179 final StoreFileInfo storeFile) throws IOException {
180 try {
181 LOG.debug("Introducing delay before adding store file to manifest");
182 Thread.sleep(2000);
183 } catch (InterruptedException ex) {
184 LOG.error("Interrupted due to error: " + ex);
185 }
186 super.storeFile(region, family, storeFile);
187 }
188 };
189 }
190
191 private Table setupTable(TableName tableName) throws Exception {
192 HTableDescriptor htd = new HTableDescriptor(tableName);
193
194 htd.setMemStoreFlushSize(5000).setConfiguration("hbase.hstore.compactionThreshold", "250");
195
196 htd.setRegionSplitPolicyClassName(ConstantSizeRegionSplitPolicy.class.getName());
197 htd.setMaxFileSize(100 * 1024 * 1024);
198
199 byte[] fam = Bytes.toBytes("fam");
200 Table table = TEST_UTIL.createTable(htd, new byte[][] {fam},
201 TEST_UTIL.getConfiguration());
202 TEST_UTIL.loadTable(table, fam);
203 return table;
204 }
205 }