1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
40 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
41 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
42 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
43 import org.apache.hadoop.hbase.util.ByteStringer;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSUtils;
46
47
48
49
50
51
52
53
54
55
56 @InterfaceAudience.Private
57 public final class SnapshotManifestV1 {
58 private static final Log LOG = LogFactory.getLog(SnapshotManifestV1.class);
59
60 public static final int DESCRIPTOR_VERSION = 0;
61
62 private SnapshotManifestV1() {
63 }
64
65 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
66 HRegionFileSystem, Path> {
67 private final Configuration conf;
68 private final Path snapshotDir;
69 private final FileSystem rootFs;
70 private final FileSystem workingDirFs;
71
72 public ManifestBuilder(final Configuration conf, final FileSystem rootFs,
73 final Path snapshotDir) throws IOException {
74 this.snapshotDir = snapshotDir;
75 this.conf = conf;
76 this.rootFs = rootFs;
77 this.workingDirFs = snapshotDir.getFileSystem(conf);
78 }
79
80 public HRegionFileSystem regionOpen(final HRegionInfo regionInfo) throws IOException {
81 HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
82 workingDirFs, snapshotDir, regionInfo);
83 return snapshotRegionFs;
84 }
85
86 public void regionClose(final HRegionFileSystem region) {
87 }
88
89 public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
90 Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
91 return familyDir;
92 }
93
94 public void familyClose(final HRegionFileSystem region, final Path family) {
95 }
96
97 public void storeFile(final HRegionFileSystem region, final Path familyDir,
98 final StoreFileInfo storeFile) throws IOException {
99 Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
100 boolean success = true;
101 if (storeFile.isReference()) {
102
103 storeFile.getReference().write(workingDirFs, referenceFile);
104 } else {
105
106
107
108
109 success = workingDirFs.createNewFile(referenceFile);
110 }
111 if (!success) {
112 throw new IOException("Failed to create reference file:" + referenceFile);
113 }
114 }
115 }
116
117 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
118 final Executor executor,final FileSystem fs, final Path snapshotDir,
119 final SnapshotDescription desc) throws IOException {
120 FileStatus[] regions = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
121 if (regions == null) {
122 LOG.debug("No regions under directory:" + snapshotDir);
123 return null;
124 }
125
126 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
127 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
128 for (final FileStatus region: regions) {
129 completionService.submit(new Callable<SnapshotRegionManifest>() {
130 @Override
131 public SnapshotRegionManifest call() throws IOException {
132 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath());
133 return buildManifestFromDisk(conf, fs, snapshotDir, hri);
134 }
135 });
136 }
137
138 ArrayList<SnapshotRegionManifest> regionsManifest =
139 new ArrayList<SnapshotRegionManifest>(regions.length);
140 try {
141 for (int i = 0; i < regions.length; ++i) {
142 regionsManifest.add(completionService.take().get());
143 }
144 } catch (InterruptedException e) {
145 throw new InterruptedIOException(e.getMessage());
146 } catch (ExecutionException e) {
147 throw new IOException(e.getCause());
148 }
149 return regionsManifest;
150 }
151
152 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
153 final SnapshotRegionManifest manifest) throws IOException {
154 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
155 fs.delete(new Path(snapshotDir, regionName), true);
156 }
157
158 static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
159 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
160 HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
161 tableDir, regionInfo, true);
162 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
163
164
165 LOG.debug("Storing region-info for snapshot.");
166 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
167
168
169 LOG.debug("Creating references for hfiles");
170
171
172
173
174
175
176 Collection<String> familyNames = regionFs.getFamilies();
177 if (familyNames != null) {
178 for (String familyName: familyNames) {
179 Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
180 if (storeFiles == null) {
181 LOG.debug("No files under family: " + familyName);
182 continue;
183 }
184
185
186 SnapshotRegionManifest.FamilyFiles.Builder family =
187 SnapshotRegionManifest.FamilyFiles.newBuilder();
188 family.setFamilyName(ByteStringer.wrap(Bytes.toBytes(familyName)));
189
190 if (LOG.isDebugEnabled()) {
191 LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
192 }
193
194
195 int i = 0;
196 int sz = storeFiles.size();
197 for (StoreFileInfo storeFile: storeFiles) {
198
199 LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
200 SnapshotRegionManifest.StoreFile.Builder sfManifest =
201 SnapshotRegionManifest.StoreFile.newBuilder();
202 sfManifest.setName(storeFile.getPath().getName());
203 family.addStoreFiles(sfManifest.build());
204 }
205 manifest.addFamilyFiles(family.build());
206 }
207 }
208 return manifest.build();
209 }
210 }