1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38 import org.apache.hadoop.hbase.testclassification.LargeTests;
39 import org.apache.hadoop.hbase.TableNotFoundException;
40 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
41 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
43 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
44 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
45 import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.FSUtils;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59
60
61
62 @Category(LargeTests.class)
63 public class TestSnapshotFromClient {
64 private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
65 protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
66 private static final int NUM_RS = 2;
67 private static final String STRING_TABLE_NAME = "test";
68 protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
69 protected static final TableName TABLE_NAME =
70 TableName.valueOf(STRING_TABLE_NAME);
71
72
73
74
75
76 @BeforeClass
77 public static void setupCluster() throws Exception {
78 setupConf(UTIL.getConfiguration());
79 UTIL.startMiniCluster(NUM_RS);
80 }
81
82 private static void setupConf(Configuration conf) {
83
84 conf.setInt("hbase.regionsever.info.port", -1);
85
86 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
87
88
89 conf.setInt("hbase.hstore.compaction.min", 10);
90 conf.setInt("hbase.hstore.compactionThreshold", 10);
91
92 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
93
94 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
95 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
96 ConstantSizeRegionSplitPolicy.class.getName());
97 }
98
99 @Before
100 public void setup() throws Exception {
101 HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
102 htd.setRegionReplication(getNumReplicas());
103 UTIL.createTable(htd, new byte[][]{TEST_FAM}, UTIL.getConfiguration());
104 }
105
106 protected int getNumReplicas() {
107 return 1;
108 }
109
110 @After
111 public void tearDown() throws Exception {
112 UTIL.deleteTable(TABLE_NAME);
113 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
114 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
115 }
116
117 @AfterClass
118 public static void cleanupTest() throws Exception {
119 try {
120 UTIL.shutdownMiniCluster();
121 } catch (Exception e) {
122 LOG.warn("failure shutting down cluster", e);
123 }
124 }
125
126
127
128
129
130 @Test (timeout=300000)
131 public void testMetaTablesSnapshot() throws Exception {
132 Admin admin = UTIL.getHBaseAdmin();
133 byte[] snapshotName = Bytes.toBytes("metaSnapshot");
134
135 try {
136 admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
137 fail("taking a snapshot of hbase:meta should not be allowed");
138 } catch (IllegalArgumentException e) {
139
140 }
141 }
142
143
144
145
146
147
148 @Test (timeout=300000)
149 public void testSnapshotDeletionWithRegex() throws Exception {
150 Admin admin = UTIL.getHBaseAdmin();
151
152 SnapshotTestingUtils.assertNoSnapshots(admin);
153
154
155 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
156 UTIL.loadTable(table, TEST_FAM);
157 table.close();
158
159 byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
160 admin.snapshot(snapshot1, TABLE_NAME);
161 LOG.debug("Snapshot1 completed.");
162
163 byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
164 admin.snapshot(snapshot2, TABLE_NAME);
165 LOG.debug("Snapshot2 completed.");
166
167 String snapshot3 = "3rdTableSnapshot";
168 admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
169 LOG.debug(snapshot3 + " completed.");
170
171
172 admin.deleteSnapshots("TableSnapshot.*");
173 List<SnapshotDescription> snapshots = admin.listSnapshots();
174 assertEquals(1, snapshots.size());
175 assertEquals(snapshots.get(0).getName(), snapshot3);
176
177 admin.deleteSnapshot(snapshot3);
178 admin.close();
179 }
180
181
182
183
184 @Test (timeout=300000)
185 public void testOfflineTableSnapshot() throws Exception {
186 Admin admin = UTIL.getHBaseAdmin();
187
188 SnapshotTestingUtils.assertNoSnapshots(admin);
189
190
191 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
192 UTIL.loadTable(table, TEST_FAM, false);
193
194 LOG.debug("FS state before disable:");
195 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
196 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
197
198
199 admin.disableTable(TABLE_NAME);
200
201 LOG.debug("FS state before snapshot:");
202 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
203 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
204
205
206 final String SNAPSHOT_NAME = "offlineTableSnapshot";
207 byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
208
209 SnapshotDescription desc = SnapshotDescription.newBuilder()
210 .setType(SnapshotDescription.Type.DISABLED)
211 .setTable(STRING_TABLE_NAME)
212 .setName(SNAPSHOT_NAME)
213 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
214 .build();
215 admin.snapshot(desc);
216 LOG.debug("Snapshot completed.");
217
218
219 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
220 snapshot, TABLE_NAME);
221
222
223 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
224 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
225 LOG.debug("FS state after snapshot:");
226 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
227 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
228
229 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
230 admin, fs);
231
232 admin.deleteSnapshot(snapshot);
233 snapshots = admin.listSnapshots();
234 SnapshotTestingUtils.assertNoSnapshots(admin);
235 }
236
237 @Test (timeout=300000)
238 public void testSnapshotFailsOnNonExistantTable() throws Exception {
239 Admin admin = UTIL.getHBaseAdmin();
240
241 SnapshotTestingUtils.assertNoSnapshots(admin);
242 String tableName = "_not_a_table";
243
244
245 boolean fail = false;
246 do {
247 try {
248 admin.getTableDescriptor(TableName.valueOf(tableName));
249 fail = true;
250 LOG.error("Table:" + tableName + " already exists, checking a new name");
251 tableName = tableName+"!";
252 } catch (TableNotFoundException e) {
253 fail = false;
254 }
255 } while (fail);
256
257
258 try {
259 admin.snapshot("fail", TableName.valueOf(tableName));
260 fail("Snapshot succeeded even though there is not table.");
261 } catch (SnapshotCreationException e) {
262 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
263 }
264 }
265
266 @Test (timeout=300000)
267 public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
268
269
270 Admin admin = UTIL.getHBaseAdmin();
271
272 SnapshotTestingUtils.assertNoSnapshots(admin);
273
274 LOG.debug("FS state before disable:");
275 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
276 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
277 admin.disableTable(TABLE_NAME);
278
279 LOG.debug("FS state before snapshot:");
280 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
281 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
282
283
284 byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
285 admin.snapshot(snapshot, TABLE_NAME);
286 LOG.debug("Snapshot completed.");
287
288
289 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
290 snapshot, TABLE_NAME);
291
292
293 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
294 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
295 LOG.debug("FS state after snapshot:");
296 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
297 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
298
299 List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM);
300 List<byte[]> nonEmptyCfs = Lists.newArrayList();
301 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
302 rootDir, admin, fs);
303
304 admin.deleteSnapshot(snapshot);
305 snapshots = admin.listSnapshots();
306 SnapshotTestingUtils.assertNoSnapshots(admin);
307 }
308
309 @Test(timeout = 300000)
310 public void testListTableSnapshots() throws Exception {
311 Admin admin = null;
312 TableName tableName2 = TableName.valueOf("testListTableSnapshots");
313 try {
314 admin = UTIL.getHBaseAdmin();
315
316 HTableDescriptor htd = new HTableDescriptor(tableName2);
317 UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
318
319 String table1Snapshot1 = "Table1Snapshot1";
320 admin.snapshot(table1Snapshot1, TABLE_NAME);
321 LOG.debug("Snapshot1 completed.");
322
323 String table1Snapshot2 = "Table1Snapshot2";
324 admin.snapshot(table1Snapshot2, TABLE_NAME);
325 LOG.debug("Snapshot2 completed.");
326
327 String table2Snapshot1 = "Table2Snapshot1";
328 admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
329 LOG.debug(table2Snapshot1 + " completed.");
330
331 List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", ".*");
332 List<String> listTableSnapshotNames = new ArrayList<String>();
333 assertEquals(3, listTableSnapshots.size());
334 for (SnapshotDescription s : listTableSnapshots) {
335 listTableSnapshotNames.add(s.getName());
336 }
337 assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
338 assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
339 assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
340 } finally {
341 if (admin != null) {
342 try {
343 admin.deleteSnapshots("Table.*");
344 } catch (SnapshotDoesNotExistException ignore) {
345 }
346 if (admin.tableExists(tableName2)) {
347 UTIL.deleteTable(tableName2);
348 }
349 admin.close();
350 }
351 }
352 }
353
354 @Test(timeout = 300000)
355 public void testListTableSnapshotsWithRegex() throws Exception {
356 Admin admin = null;
357 try {
358 admin = UTIL.getHBaseAdmin();
359
360 String table1Snapshot1 = "Table1Snapshot1";
361 admin.snapshot(table1Snapshot1, TABLE_NAME);
362 LOG.debug("Snapshot1 completed.");
363
364 String table1Snapshot2 = "Table1Snapshot2";
365 admin.snapshot(table1Snapshot2, TABLE_NAME);
366 LOG.debug("Snapshot2 completed.");
367
368 String table2Snapshot1 = "Table2Snapshot1";
369 admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
370 LOG.debug(table2Snapshot1 + " completed.");
371
372 List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", "Table1.*");
373 List<String> listTableSnapshotNames = new ArrayList<String>();
374 assertEquals(2, listTableSnapshots.size());
375 for (SnapshotDescription s : listTableSnapshots) {
376 listTableSnapshotNames.add(s.getName());
377 }
378 assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
379 assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
380 assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
381 } finally {
382 if (admin != null) {
383 try {
384 admin.deleteSnapshots("Table.*");
385 } catch (SnapshotDoesNotExistException ignore) {
386 }
387 admin.close();
388 }
389 }
390 }
391
392 @Test(timeout = 300000)
393 public void testDeleteTableSnapshots() throws Exception {
394 Admin admin = null;
395 TableName tableName2 = TableName.valueOf("testListTableSnapshots");
396 try {
397 admin = UTIL.getHBaseAdmin();
398
399 HTableDescriptor htd = new HTableDescriptor(tableName2);
400 UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
401
402 String table1Snapshot1 = "Table1Snapshot1";
403 admin.snapshot(table1Snapshot1, TABLE_NAME);
404 LOG.debug("Snapshot1 completed.");
405
406 String table1Snapshot2 = "Table1Snapshot2";
407 admin.snapshot(table1Snapshot2, TABLE_NAME);
408 LOG.debug("Snapshot2 completed.");
409
410 String table2Snapshot1 = "Table2Snapshot1";
411 admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
412 LOG.debug(table2Snapshot1 + " completed.");
413
414 admin.deleteTableSnapshots("test.*", ".*");
415 assertEquals(0, admin.listTableSnapshots("test.*", ".*").size());
416 } finally {
417 if (admin != null) {
418 if (admin.tableExists(tableName2)) {
419 UTIL.deleteTable(tableName2);
420 }
421 admin.close();
422 }
423 }
424 }
425
426 @Test(timeout = 300000)
427 public void testDeleteTableSnapshotsWithRegex() throws Exception {
428 Admin admin = null;
429 try {
430 admin = UTIL.getHBaseAdmin();
431
432 String table1Snapshot1 = "Table1Snapshot1";
433 admin.snapshot(table1Snapshot1, TABLE_NAME);
434 LOG.debug("Snapshot1 completed.");
435
436 String table1Snapshot2 = "Table1Snapshot2";
437 admin.snapshot(table1Snapshot2, TABLE_NAME);
438 LOG.debug("Snapshot2 completed.");
439
440 String table2Snapshot1 = "Table2Snapshot1";
441 admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
442 LOG.debug(table2Snapshot1 + " completed.");
443
444 admin.deleteTableSnapshots("test.*", "Table1.*");
445 assertEquals(1, admin.listTableSnapshots("test.*", ".*").size());
446 } finally {
447 if (admin != null) {
448 try {
449 admin.deleteTableSnapshots("test.*", ".*");
450 } catch (SnapshotDoesNotExistException ignore) {
451 }
452 admin.close();
453 }
454 }
455 }
456 }