1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.fs.FSDataOutputStream;
24 import org.apache.hadoop.fs.FileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HColumnDescriptor;
27 import org.apache.hadoop.hbase.HRegionInfo;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
31 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
32 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
33 import org.apache.hadoop.hbase.testclassification.MasterTests;
34 import org.apache.hadoop.hbase.testclassification.SmallTests;
35 import org.apache.hadoop.hbase.util.ByteStringer;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.junit.After;
38 import org.junit.Before;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 import java.io.IOException;
43
44 import static org.junit.Assert.fail;
45
46 @Category({MasterTests.class, SmallTests.class})
47 public class TestSnapshotManifest {
48 private final Log LOG = LogFactory.getLog(getClass());
49
50 private static final String TABLE_NAME_STR = "testSnapshotManifest";
51 private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
52 private static final int TEST_NUM_REGIONS = 16000;
53 private static final int TEST_NUM_REGIONFILES = 1000000;
54
55 private static HBaseTestingUtility TEST_UTIL;
56 private Configuration conf;
57 private FileSystem fs;
58 private Path rootDir;
59 private Path snapshotDir;
60 private SnapshotDescription snapshotDesc;
61 private SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder;
62
63 @Before
64 public void setup() throws Exception {
65 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
66
67 rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
68 fs = TEST_UTIL.getTestFileSystem();
69 conf = TEST_UTIL.getConfiguration();
70
71 SnapshotTestingUtils.SnapshotMock snapshotMock =
72 new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
73 builder = snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
74 snapshotDir = builder.commit();
75 snapshotDesc = builder.getSnapshotDescription();
76 }
77
78 @After
79 public void tearDown() throws Exception {
80 fs.delete(rootDir,true);
81 }
82
83 @Test
84 public void testReadSnapshotManifest() throws IOException {
85
86 Path p = createDataManifest();
87 try {
88 SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
89 fail("fail to test snapshot manifest because message size is too small.");
90 } catch (CorruptedSnapshotException cse) {
91 try {
92 conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
93 SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
94 LOG.info("open snapshot manifest succeed.");
95 } catch (CorruptedSnapshotException cse2) {
96 fail("fail to take snapshot because Manifest proto-message too large.");
97 }
98 } finally {
99 fs.delete(p, false);
100 }
101 }
102
103 @Test
104 public void testReadSnapshotRegionManifest() throws IOException {
105
106
107 fs.delete(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME), true);
108 Path regionPath = createRegionManifest();
109
110 try {
111 conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
112 SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
113 } catch (CorruptedSnapshotException e) {
114 fail("fail to test snapshot manifest because region message size is too small.");
115 } finally {
116 fs.delete(regionPath, false);
117 }
118 }
119
120 private Path createDataManifest() throws IOException {
121 SnapshotDataManifest.Builder dataManifestBuilder =
122 SnapshotDataManifest.newBuilder();
123 byte[] startKey = null;
124 byte[] stopKey = null;
125 for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
126 stopKey = Bytes.toBytes(String.format("%016d", i));
127 HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
128 SnapshotRegionManifest.Builder dataRegionManifestBuilder =
129 SnapshotRegionManifest.newBuilder();
130
131 for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) {
132 SnapshotRegionManifest.FamilyFiles.Builder family =
133 SnapshotRegionManifest.FamilyFiles.newBuilder();
134 family.setFamilyName(ByteStringer.wrap(hcd.getName()));
135 for (int j = 0; j < 100; ++j) {
136 SnapshotRegionManifest.StoreFile.Builder sfManifest =
137 SnapshotRegionManifest.StoreFile.newBuilder();
138 sfManifest.setName(String.format("%032d", i));
139 sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
140 family.addStoreFiles(sfManifest.build());
141 }
142 dataRegionManifestBuilder.addFamilyFiles(family.build());
143 }
144
145 dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
146 dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
147
148 startKey = stopKey;
149 }
150
151 dataManifestBuilder.setTableSchema(builder.getTableDescriptor().convert());
152
153 SnapshotDataManifest dataManifest = dataManifestBuilder.build();
154 return writeDataManifest(dataManifest);
155 }
156
157 private Path createRegionManifest() throws IOException {
158 byte[] startKey = Bytes.toBytes("AAAAAA");
159 byte[] stopKey = Bytes.toBytes("BBBBBB");
160 HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
161 SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder();
162 dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
163
164 for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) {
165 SnapshotRegionManifest.FamilyFiles.Builder family =
166 SnapshotRegionManifest.FamilyFiles.newBuilder();
167 family.setFamilyName(ByteStringer.wrap(hcd.getName()));
168 for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) {
169 SnapshotRegionManifest.StoreFile.Builder sfManifest =
170 SnapshotRegionManifest.StoreFile.newBuilder();
171 sfManifest.setName(String.format("%064d", j));
172 sfManifest.setFileSize(j * 1024);
173 family.addStoreFiles(sfManifest.build());
174 }
175 dataRegionManifestBuilder.addFamilyFiles(family.build());
176 }
177
178 SnapshotRegionManifest manifest = dataRegionManifestBuilder.build();
179 Path regionPath = new Path(snapshotDir,
180 SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName());
181
182 FSDataOutputStream stream = fs.create(regionPath);
183 try {
184 manifest.writeTo(stream);
185 } finally {
186 stream.close();
187 }
188
189 return regionPath;
190 }
191
192 private Path writeDataManifest(final SnapshotDataManifest manifest)
193 throws IOException {
194 Path dataRegionPath = new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME);
195 FSDataOutputStream stream = fs.create(dataRegionPath);
196 try {
197 manifest.writeTo(stream);
198 } finally {
199 stream.close();
200 }
201
202 return dataRegionPath;
203 }
204 }