View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication;
20  
21  import com.google.protobuf.ByteString;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.hbase.TableName;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.classification.InterfaceStability;
28  import org.apache.hadoop.hbase.exceptions.DeserializationException;
29  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
30  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
31  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.apache.hadoop.hbase.util.Strings;
34  
35  import java.io.IOException;
36  import java.util.ArrayList;
37  import java.util.Collection;
38  import java.util.HashMap;
39  import java.util.List;
40  import java.util.Map;
41  
42  @InterfaceAudience.Private
43  @InterfaceStability.Stable
44  public final class ReplicationSerDeHelper {
45    private static final Log LOG = LogFactory.getLog(ReplicationSerDeHelper.class);
46  
47    private ReplicationSerDeHelper() {}
48  
49    /** convert map to TableCFs Object */
50    public static ZooKeeperProtos.TableCF[] convert(
51        Map<TableName, ? extends Collection<String>> tableCfs) {
52      if (tableCfs == null) {
53        return null;
54      }
55      List<ZooKeeperProtos.TableCF> tableCFList = new ArrayList<>();
56      ZooKeeperProtos.TableCF.Builder tableCFBuilder =  ZooKeeperProtos.TableCF.newBuilder();
57      for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
58        tableCFBuilder.clear();
59        tableCFBuilder.setTableName(ProtobufUtil.toProtoTableName(entry.getKey()));
60        Collection<String> v = entry.getValue();
61        if (v != null && !v.isEmpty()) {
62          for (String value : entry.getValue()) {
63            tableCFBuilder.addFamilies(ByteString.copyFromUtf8(value));
64          }
65        }
66        tableCFList.add(tableCFBuilder.build());
67      }
68      return tableCFList.toArray(new ZooKeeperProtos.TableCF[tableCFList.size()]);
69    }
70  
71    public static String convertToString(Map<TableName, ? extends Collection<String>> tableCfs) {
72      if (tableCfs == null) {
73        return null;
74      }
75      return convert(convert(tableCfs));
76    }
77  
78    /**
79     *  Convert string to TableCFs Object.
80     *  This is only for read TableCFs information from TableCF node.
81     *  Input String Format: ns1.table1:cf1,cf2;ns2.table2:cfA,cfB;ns3.table3.
82     * */
83    public static ZooKeeperProtos.TableCF[] convert(String tableCFsConfig) {
84      if (tableCFsConfig == null || tableCFsConfig.trim().length() == 0) {
85        return null;
86      }
87      List<ZooKeeperProtos.TableCF> tableCFList = new ArrayList<>();
88      ZooKeeperProtos.TableCF.Builder tableCFBuilder = ZooKeeperProtos.TableCF.newBuilder();
89  
90      String[] tables = tableCFsConfig.split(";");
91      for (String tab : tables) {
92        // 1 ignore empty table config
93        tab = tab.trim();
94        if (tab.length() == 0) {
95          continue;
96        }
97        // 2 split to "table" and "cf1,cf2"
98        //   for each table: "table#cf1,cf2" or "table"
99        String[] pair = tab.split(":");
100       String tabName = pair[0].trim();
101       if (pair.length > 2 || tabName.length() == 0) {
102         LOG.info("incorrect format:" + tableCFsConfig);
103         continue;
104       }
105 
106       tableCFBuilder.clear();
107       // split namespace from tableName
108       String ns = "default";
109       String tName = tabName;
110       String[] dbs = tabName.split("\\.");
111       if (dbs != null && dbs.length == 2) {
112         ns = dbs[0];
113         tName = dbs[1];
114       }
115       tableCFBuilder.setTableName(
116         ProtobufUtil.toProtoTableName(TableName.valueOf(ns, tName)));
117 
118       // 3 parse "cf1,cf2" part to List<cf>
119       if (pair.length == 2) {
120         String[] cfsList = pair[1].split(",");
121         for (String cf : cfsList) {
122           String cfName = cf.trim();
123           if (cfName.length() > 0) {
124             tableCFBuilder.addFamilies(ByteString.copyFromUtf8(cfName));
125           }
126         }
127       }
128       tableCFList.add(tableCFBuilder.build());
129     }
130     return tableCFList.toArray(new ZooKeeperProtos.TableCF[tableCFList.size()]);
131   }
132 
133   /**
134    *  Convert TableCFs Object to String.
135    *  Output String Format: ns1.table1:cf1,cf2;ns2.table2:cfA,cfB;table3
136    * */
137   public static String convert(ZooKeeperProtos.TableCF[] tableCFs) {
138     StringBuilder sb = new StringBuilder();
139     for (int i = 0, n = tableCFs.length; i < n; i++) {
140       ZooKeeperProtos.TableCF tableCF = tableCFs[i];
141       String namespace = tableCF.getTableName().getNamespace().toStringUtf8();
142       if (!Strings.isEmpty(namespace)) {
143         sb.append(namespace).append(".").
144             append(tableCF.getTableName().getQualifier().toStringUtf8())
145             .append(":");
146       } else {
147         sb.append(tableCF.getTableName().toString()).append(":");
148       }
149       for (int j = 0; j < tableCF.getFamiliesCount(); j++) {
150         sb.append(tableCF.getFamilies(j).toStringUtf8()).append(",");
151       }
152       sb.deleteCharAt(sb.length() - 1).append(";");
153     }
154     if (sb.length() > 0) {
155       sb.deleteCharAt(sb.length() - 1);
156     }
157     return sb.toString();
158   }
159 
160   /**
161    *  Get TableCF in TableCFs, if not exist, return null.
162    * */
163   public static ZooKeeperProtos.TableCF getTableCF(ZooKeeperProtos.TableCF[] tableCFs,
164                                            String table) {
165     for (int i = 0, n = tableCFs.length; i < n; i++) {
166       ZooKeeperProtos.TableCF tableCF = tableCFs[i];
167       if (tableCF.getTableName().getQualifier().toStringUtf8().equals(table)) {
168         return tableCF;
169       }
170     }
171     return null;
172   }
173 
174   /**
175    *  Parse bytes into TableCFs.
176    *  It is used for backward compatibility.
177    *  Old format bytes have no PB_MAGIC Header
178    * */
179   public static ZooKeeperProtos.TableCF[] parseTableCFs(byte[] bytes) throws IOException {
180     if (bytes == null) {
181       return null;
182     }
183     return ReplicationSerDeHelper.convert(Bytes.toString(bytes));
184   }
185 
186   /**
187    *  Convert tableCFs string into Map.
188    * */
189   public static Map<TableName, List<String>> parseTableCFsFromConfig(String tableCFsConfig) {
190     ZooKeeperProtos.TableCF[] tableCFs = convert(tableCFsConfig);
191     return convert2Map(tableCFs);
192   }
193 
194   /**
195    *  Convert tableCFs Object to Map.
196    * */
197   public static Map<TableName, List<String>> convert2Map(ZooKeeperProtos.TableCF[] tableCFs) {
198     if (tableCFs == null || tableCFs.length == 0) {
199       return null;
200     }
201     Map<TableName, List<String>> tableCFsMap = new HashMap<TableName, List<String>>();
202     for (int i = 0, n = tableCFs.length; i < n; i++) {
203       ZooKeeperProtos.TableCF tableCF = tableCFs[i];
204       List<String> families = new ArrayList<>();
205       for (int j = 0, m = tableCF.getFamiliesCount(); j < m; j++) {
206         families.add(tableCF.getFamilies(j).toStringUtf8());
207       }
208       if (families.size() > 0) {
209         tableCFsMap.put(ProtobufUtil.toTableName(tableCF.getTableName()), families);
210       } else {
211         tableCFsMap.put(ProtobufUtil.toTableName(tableCF.getTableName()), null);
212       }
213     }
214 
215     return tableCFsMap;
216   }
217 
218   /**
219    * @param bytes Content of a peer znode.
220    * @return ClusterKey parsed from the passed bytes.
221    * @throws DeserializationException
222    */
223   public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
224       throws DeserializationException {
225     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
226       int pblen = ProtobufUtil.lengthOfPBMagic();
227       ZooKeeperProtos.ReplicationPeer.Builder builder =
228           ZooKeeperProtos.ReplicationPeer.newBuilder();
229       ZooKeeperProtos.ReplicationPeer peer;
230       try {
231         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
232         peer = builder.build();
233       } catch (IOException e) {
234         throw new DeserializationException(e);
235       }
236       return convert(peer);
237     } else {
238       if (bytes.length > 0) {
239         return new ReplicationPeerConfig().setClusterKey(Bytes.toString(bytes));
240       }
241       return new ReplicationPeerConfig().setClusterKey("");
242     }
243   }
244 
245   private static ReplicationPeerConfig convert(ZooKeeperProtos.ReplicationPeer peer) {
246     ReplicationPeerConfig peerConfig = new ReplicationPeerConfig();
247     if (peer.hasClusterkey()) {
248       peerConfig.setClusterKey(peer.getClusterkey());
249     }
250     if (peer.hasReplicationEndpointImpl()) {
251       peerConfig.setReplicationEndpointImpl(peer.getReplicationEndpointImpl());
252     }
253 
254     for (HBaseProtos.BytesBytesPair pair : peer.getDataList()) {
255       peerConfig.getPeerData().put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
256     }
257 
258     for (HBaseProtos.NameStringPair pair : peer.getConfigurationList()) {
259       peerConfig.getConfiguration().put(pair.getName(), pair.getValue());
260     }
261 
262     Map<TableName, ? extends Collection<String>> tableCFsMap = convert2Map(
263       peer.getTableCfsList().toArray(new ZooKeeperProtos.TableCF[peer.getTableCfsCount()]));
264     if (tableCFsMap != null) {
265       peerConfig.setTableCFsMap(tableCFsMap);
266     }
267 
268     if (peer.hasBandwidth()) {
269       peerConfig.setBandwidth(peer.getBandwidth());
270     }
271     return peerConfig;
272   }
273 
274   /**
275    * @param peerConfig
276    * @return Serialized protobuf of <code>peerConfig</code> with pb magic prefix prepended suitable
277    *         for use as content of a this.peersZNode; i.e. the content of PEER_ID znode under
278    *         /hbase/replication/peers/PEER_ID
279    */
280   public static byte[] toByteArray(final ReplicationPeerConfig peerConfig) {
281     byte[] bytes = convert(peerConfig).toByteArray();
282     return ProtobufUtil.prependPBMagic(bytes);
283   }
284 
285   private static ZooKeeperProtos.ReplicationPeer convert(ReplicationPeerConfig  peerConfig) {
286     ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
287     if (peerConfig.getClusterKey() != null) {
288       builder.setClusterkey(peerConfig.getClusterKey());
289     }
290     if (peerConfig.getReplicationEndpointImpl() != null) {
291       builder.setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl());
292     }
293 
294     for (Map.Entry<byte[], byte[]> entry : peerConfig.getPeerData().entrySet()) {
295       builder.addData(HBaseProtos.BytesBytesPair.newBuilder()
296           .setFirst(ByteString.copyFrom(entry.getKey()))
297           .setSecond(ByteString.copyFrom(entry.getValue()))
298           .build());
299     }
300 
301     for (Map.Entry<String, String> entry : peerConfig.getConfiguration().entrySet()) {
302       builder.addConfiguration(HBaseProtos.NameStringPair.newBuilder()
303           .setName(entry.getKey())
304           .setValue(entry.getValue())
305           .build());
306     }
307 
308     ZooKeeperProtos.TableCF[] tableCFs = convert(peerConfig.getTableCFsMap());
309     if (tableCFs != null) {
310       for (int i = 0; i < tableCFs.length; i++) {
311         builder.addTableCfs(tableCFs[i]);
312       }
313     }
314 
315     builder.setBandwidth(peerConfig.getBandwidth());
316     return builder.build();
317   }
318 }