1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication;
20
21 import com.google.protobuf.ByteString;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.TableName;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.classification.InterfaceStability;
28 import org.apache.hadoop.hbase.exceptions.DeserializationException;
29 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
30 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
31 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.hbase.util.Strings;
34
35 import java.io.IOException;
36 import java.util.ArrayList;
37 import java.util.Collection;
38 import java.util.HashMap;
39 import java.util.List;
40 import java.util.Map;
41
42 @InterfaceAudience.Private
43 @InterfaceStability.Stable
44 public final class ReplicationSerDeHelper {
45 private static final Log LOG = LogFactory.getLog(ReplicationSerDeHelper.class);
46
47 private ReplicationSerDeHelper() {}
48
49
50 public static ZooKeeperProtos.TableCF[] convert(
51 Map<TableName, ? extends Collection<String>> tableCfs) {
52 if (tableCfs == null) {
53 return null;
54 }
55 List<ZooKeeperProtos.TableCF> tableCFList = new ArrayList<>();
56 ZooKeeperProtos.TableCF.Builder tableCFBuilder = ZooKeeperProtos.TableCF.newBuilder();
57 for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
58 tableCFBuilder.clear();
59 tableCFBuilder.setTableName(ProtobufUtil.toProtoTableName(entry.getKey()));
60 Collection<String> v = entry.getValue();
61 if (v != null && !v.isEmpty()) {
62 for (String value : entry.getValue()) {
63 tableCFBuilder.addFamilies(ByteString.copyFromUtf8(value));
64 }
65 }
66 tableCFList.add(tableCFBuilder.build());
67 }
68 return tableCFList.toArray(new ZooKeeperProtos.TableCF[tableCFList.size()]);
69 }
70
71 public static String convertToString(Map<TableName, ? extends Collection<String>> tableCfs) {
72 if (tableCfs == null) {
73 return null;
74 }
75 return convert(convert(tableCfs));
76 }
77
78
79
80
81
82
83 public static ZooKeeperProtos.TableCF[] convert(String tableCFsConfig) {
84 if (tableCFsConfig == null || tableCFsConfig.trim().length() == 0) {
85 return null;
86 }
87 List<ZooKeeperProtos.TableCF> tableCFList = new ArrayList<>();
88 ZooKeeperProtos.TableCF.Builder tableCFBuilder = ZooKeeperProtos.TableCF.newBuilder();
89
90 String[] tables = tableCFsConfig.split(";");
91 for (String tab : tables) {
92
93 tab = tab.trim();
94 if (tab.length() == 0) {
95 continue;
96 }
97
98
99 String[] pair = tab.split(":");
100 String tabName = pair[0].trim();
101 if (pair.length > 2 || tabName.length() == 0) {
102 LOG.info("incorrect format:" + tableCFsConfig);
103 continue;
104 }
105
106 tableCFBuilder.clear();
107
108 String ns = "default";
109 String tName = tabName;
110 String[] dbs = tabName.split("\\.");
111 if (dbs != null && dbs.length == 2) {
112 ns = dbs[0];
113 tName = dbs[1];
114 }
115 tableCFBuilder.setTableName(
116 ProtobufUtil.toProtoTableName(TableName.valueOf(ns, tName)));
117
118
119 if (pair.length == 2) {
120 String[] cfsList = pair[1].split(",");
121 for (String cf : cfsList) {
122 String cfName = cf.trim();
123 if (cfName.length() > 0) {
124 tableCFBuilder.addFamilies(ByteString.copyFromUtf8(cfName));
125 }
126 }
127 }
128 tableCFList.add(tableCFBuilder.build());
129 }
130 return tableCFList.toArray(new ZooKeeperProtos.TableCF[tableCFList.size()]);
131 }
132
133
134
135
136
137 public static String convert(ZooKeeperProtos.TableCF[] tableCFs) {
138 StringBuilder sb = new StringBuilder();
139 for (int i = 0, n = tableCFs.length; i < n; i++) {
140 ZooKeeperProtos.TableCF tableCF = tableCFs[i];
141 String namespace = tableCF.getTableName().getNamespace().toStringUtf8();
142 if (!Strings.isEmpty(namespace)) {
143 sb.append(namespace).append(".").
144 append(tableCF.getTableName().getQualifier().toStringUtf8())
145 .append(":");
146 } else {
147 sb.append(tableCF.getTableName().toString()).append(":");
148 }
149 for (int j = 0; j < tableCF.getFamiliesCount(); j++) {
150 sb.append(tableCF.getFamilies(j).toStringUtf8()).append(",");
151 }
152 sb.deleteCharAt(sb.length() - 1).append(";");
153 }
154 if (sb.length() > 0) {
155 sb.deleteCharAt(sb.length() - 1);
156 }
157 return sb.toString();
158 }
159
160
161
162
163 public static ZooKeeperProtos.TableCF getTableCF(ZooKeeperProtos.TableCF[] tableCFs,
164 String table) {
165 for (int i = 0, n = tableCFs.length; i < n; i++) {
166 ZooKeeperProtos.TableCF tableCF = tableCFs[i];
167 if (tableCF.getTableName().getQualifier().toStringUtf8().equals(table)) {
168 return tableCF;
169 }
170 }
171 return null;
172 }
173
174
175
176
177
178
179 public static ZooKeeperProtos.TableCF[] parseTableCFs(byte[] bytes) throws IOException {
180 if (bytes == null) {
181 return null;
182 }
183 return ReplicationSerDeHelper.convert(Bytes.toString(bytes));
184 }
185
186
187
188
189 public static Map<TableName, List<String>> parseTableCFsFromConfig(String tableCFsConfig) {
190 ZooKeeperProtos.TableCF[] tableCFs = convert(tableCFsConfig);
191 return convert2Map(tableCFs);
192 }
193
194
195
196
197 public static Map<TableName, List<String>> convert2Map(ZooKeeperProtos.TableCF[] tableCFs) {
198 if (tableCFs == null || tableCFs.length == 0) {
199 return null;
200 }
201 Map<TableName, List<String>> tableCFsMap = new HashMap<TableName, List<String>>();
202 for (int i = 0, n = tableCFs.length; i < n; i++) {
203 ZooKeeperProtos.TableCF tableCF = tableCFs[i];
204 List<String> families = new ArrayList<>();
205 for (int j = 0, m = tableCF.getFamiliesCount(); j < m; j++) {
206 families.add(tableCF.getFamilies(j).toStringUtf8());
207 }
208 if (families.size() > 0) {
209 tableCFsMap.put(ProtobufUtil.toTableName(tableCF.getTableName()), families);
210 } else {
211 tableCFsMap.put(ProtobufUtil.toTableName(tableCF.getTableName()), null);
212 }
213 }
214
215 return tableCFsMap;
216 }
217
218
219
220
221
222
223 public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
224 throws DeserializationException {
225 if (ProtobufUtil.isPBMagicPrefix(bytes)) {
226 int pblen = ProtobufUtil.lengthOfPBMagic();
227 ZooKeeperProtos.ReplicationPeer.Builder builder =
228 ZooKeeperProtos.ReplicationPeer.newBuilder();
229 ZooKeeperProtos.ReplicationPeer peer;
230 try {
231 ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
232 peer = builder.build();
233 } catch (IOException e) {
234 throw new DeserializationException(e);
235 }
236 return convert(peer);
237 } else {
238 if (bytes.length > 0) {
239 return new ReplicationPeerConfig().setClusterKey(Bytes.toString(bytes));
240 }
241 return new ReplicationPeerConfig().setClusterKey("");
242 }
243 }
244
245 private static ReplicationPeerConfig convert(ZooKeeperProtos.ReplicationPeer peer) {
246 ReplicationPeerConfig peerConfig = new ReplicationPeerConfig();
247 if (peer.hasClusterkey()) {
248 peerConfig.setClusterKey(peer.getClusterkey());
249 }
250 if (peer.hasReplicationEndpointImpl()) {
251 peerConfig.setReplicationEndpointImpl(peer.getReplicationEndpointImpl());
252 }
253
254 for (HBaseProtos.BytesBytesPair pair : peer.getDataList()) {
255 peerConfig.getPeerData().put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
256 }
257
258 for (HBaseProtos.NameStringPair pair : peer.getConfigurationList()) {
259 peerConfig.getConfiguration().put(pair.getName(), pair.getValue());
260 }
261
262 Map<TableName, ? extends Collection<String>> tableCFsMap = convert2Map(
263 peer.getTableCfsList().toArray(new ZooKeeperProtos.TableCF[peer.getTableCfsCount()]));
264 if (tableCFsMap != null) {
265 peerConfig.setTableCFsMap(tableCFsMap);
266 }
267
268 if (peer.hasBandwidth()) {
269 peerConfig.setBandwidth(peer.getBandwidth());
270 }
271 return peerConfig;
272 }
273
274
275
276
277
278
279
280 public static byte[] toByteArray(final ReplicationPeerConfig peerConfig) {
281 byte[] bytes = convert(peerConfig).toByteArray();
282 return ProtobufUtil.prependPBMagic(bytes);
283 }
284
285 private static ZooKeeperProtos.ReplicationPeer convert(ReplicationPeerConfig peerConfig) {
286 ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
287 if (peerConfig.getClusterKey() != null) {
288 builder.setClusterkey(peerConfig.getClusterKey());
289 }
290 if (peerConfig.getReplicationEndpointImpl() != null) {
291 builder.setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl());
292 }
293
294 for (Map.Entry<byte[], byte[]> entry : peerConfig.getPeerData().entrySet()) {
295 builder.addData(HBaseProtos.BytesBytesPair.newBuilder()
296 .setFirst(ByteString.copyFrom(entry.getKey()))
297 .setSecond(ByteString.copyFrom(entry.getValue()))
298 .build());
299 }
300
301 for (Map.Entry<String, String> entry : peerConfig.getConfiguration().entrySet()) {
302 builder.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
303 .setName(entry.getKey())
304 .setValue(entry.getValue())
305 .build());
306 }
307
308 ZooKeeperProtos.TableCF[] tableCFs = convert(peerConfig.getTableCFsMap());
309 if (tableCFs != null) {
310 for (int i = 0; i < tableCFs.length; i++) {
311 builder.addTableCfs(tableCFs[i]);
312 }
313 }
314
315 builder.setBandwidth(peerConfig.getBandwidth());
316 return builder.build();
317 }
318 }