001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.replication;
020
021import java.util.Collection;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Set;
027import java.util.TreeMap;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.util.Bytes;
030import org.apache.yetus.audience.InterfaceAudience;
031
032import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
033
034/**
035 * A configuration for the replication peer cluster.
036 */
037@InterfaceAudience.Public
038public class ReplicationPeerConfig {
039
040  private String clusterKey;
041  private String replicationEndpointImpl;
042  private final Map<byte[], byte[]> peerData;
043  private final Map<String, String> configuration;
044  private Map<TableName, ? extends Collection<String>> tableCFsMap = null;
045  private Set<String> namespaces = null;
046  // Default value is true, means replicate all user tables to peer cluster.
047  private boolean replicateAllUserTables = true;
048  private Map<TableName, ? extends Collection<String>> excludeTableCFsMap = null;
049  private Set<String> excludeNamespaces = null;
050  private long bandwidth = 0;
051  private final boolean serial;
052
053  private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) {
054    this.clusterKey = builder.clusterKey;
055    this.replicationEndpointImpl = builder.replicationEndpointImpl;
056    this.peerData = Collections.unmodifiableMap(builder.peerData);
057    this.configuration = Collections.unmodifiableMap(builder.configuration);
058    this.tableCFsMap =
059        builder.tableCFsMap != null ? unmodifiableTableCFsMap(builder.tableCFsMap) : null;
060    this.namespaces =
061        builder.namespaces != null ? Collections.unmodifiableSet(builder.namespaces) : null;
062    this.replicateAllUserTables = builder.replicateAllUserTables;
063    this.excludeTableCFsMap =
064        builder.excludeTableCFsMap != null ? unmodifiableTableCFsMap(builder.excludeTableCFsMap)
065            : null;
066    this.excludeNamespaces =
067        builder.excludeNamespaces != null ? Collections.unmodifiableSet(builder.excludeNamespaces)
068            : null;
069    this.bandwidth = builder.bandwidth;
070    this.serial = builder.serial;
071  }
072
073  private Map<TableName, List<String>>
074      unmodifiableTableCFsMap(Map<TableName, List<String>> tableCFsMap) {
075    Map<TableName, List<String>> newTableCFsMap = new HashMap<>();
076    tableCFsMap.forEach((table, cfs) -> newTableCFsMap.put(table,
077      cfs != null ? Collections.unmodifiableList(cfs) : null));
078    return Collections.unmodifiableMap(newTableCFsMap);
079  }
080
081  /**
082   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
083   *             {@link ReplicationPeerConfigBuilder} to create new ReplicationPeerConfig.
084   */
085  @Deprecated
086  public ReplicationPeerConfig() {
087    this.peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR);
088    this.configuration = new HashMap<>(0);
089    this.serial = false;
090  }
091
092  /**
093   * Set the clusterKey which is the concatenation of the slave cluster's:
094   * hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
095   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
096   *             {@link ReplicationPeerConfigBuilder#setClusterKey(String)} instead.
097   */
098  @Deprecated
099  public ReplicationPeerConfig setClusterKey(String clusterKey) {
100    this.clusterKey = clusterKey;
101    return this;
102  }
103
104  /**
105   * Sets the ReplicationEndpoint plugin class for this peer.
106   * @param replicationEndpointImpl a class implementing ReplicationEndpoint
107   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
108   *             {@link ReplicationPeerConfigBuilder#setReplicationEndpointImpl(String)} instead.
109   */
110  @Deprecated
111  public ReplicationPeerConfig setReplicationEndpointImpl(String replicationEndpointImpl) {
112    this.replicationEndpointImpl = replicationEndpointImpl;
113    return this;
114  }
115
116  public String getClusterKey() {
117    return clusterKey;
118  }
119
120  public String getReplicationEndpointImpl() {
121    return replicationEndpointImpl;
122  }
123
124  public Map<byte[], byte[]> getPeerData() {
125    return peerData;
126  }
127
128  public Map<String, String> getConfiguration() {
129    return configuration;
130  }
131
132  public Map<TableName, List<String>> getTableCFsMap() {
133    return (Map<TableName, List<String>>) tableCFsMap;
134  }
135
136  /**
137   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
138   *             {@link ReplicationPeerConfigBuilder#setTableCFsMap(Map)} instead.
139   */
140  @Deprecated
141  public ReplicationPeerConfig setTableCFsMap(Map<TableName,
142                                              ? extends Collection<String>> tableCFsMap) {
143    this.tableCFsMap = tableCFsMap;
144    return this;
145  }
146
147  public Set<String> getNamespaces() {
148    return this.namespaces;
149  }
150
151  /**
152   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
153   *             {@link ReplicationPeerConfigBuilder#setNamespaces(Set)} instead.
154   */
155  @Deprecated
156  public ReplicationPeerConfig setNamespaces(Set<String> namespaces) {
157    this.namespaces = namespaces;
158    return this;
159  }
160
161  public long getBandwidth() {
162    return this.bandwidth;
163  }
164
165  /**
166   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
167   *             {@link ReplicationPeerConfigBuilder#setBandwidth(long)} instead.
168   */
169  @Deprecated
170  public ReplicationPeerConfig setBandwidth(long bandwidth) {
171    this.bandwidth = bandwidth;
172    return this;
173  }
174
175  public boolean replicateAllUserTables() {
176    return this.replicateAllUserTables;
177  }
178
179  /**
180   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
181   *             {@link ReplicationPeerConfigBuilder#setReplicateAllUserTables(boolean)} instead.
182   */
183  @Deprecated
184  public ReplicationPeerConfig setReplicateAllUserTables(boolean replicateAllUserTables) {
185    this.replicateAllUserTables = replicateAllUserTables;
186    return this;
187  }
188
189  public Map<TableName, List<String>> getExcludeTableCFsMap() {
190    return (Map<TableName, List<String>>) excludeTableCFsMap;
191  }
192
193  /**
194   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
195   *             {@link ReplicationPeerConfigBuilder#setExcludeTableCFsMap(Map)} instead.
196   */
197  @Deprecated
198  public ReplicationPeerConfig setExcludeTableCFsMap(Map<TableName,
199                                              ? extends Collection<String>> tableCFsMap) {
200    this.excludeTableCFsMap = tableCFsMap;
201    return this;
202  }
203
204  public Set<String> getExcludeNamespaces() {
205    return this.excludeNamespaces;
206  }
207
208  /**
209   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
210   *             {@link ReplicationPeerConfigBuilder#setExcludeNamespaces(Set)} instead.
211   */
212  @Deprecated
213  public ReplicationPeerConfig setExcludeNamespaces(Set<String> namespaces) {
214    this.excludeNamespaces = namespaces;
215    return this;
216  }
217
218  public static ReplicationPeerConfigBuilder newBuilder() {
219    return new ReplicationPeerConfigBuilderImpl();
220  }
221
222  public boolean isSerial() {
223    return serial;
224  }
225
226  public static ReplicationPeerConfigBuilder newBuilder(ReplicationPeerConfig peerConfig) {
227    ReplicationPeerConfigBuilderImpl builder = new ReplicationPeerConfigBuilderImpl();
228    builder.setClusterKey(peerConfig.getClusterKey())
229      .setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl())
230      .putAllPeerData(peerConfig.getPeerData()).putAllConfiguration(peerConfig.getConfiguration())
231      .setTableCFsMap(peerConfig.getTableCFsMap()).setNamespaces(peerConfig.getNamespaces())
232      .setReplicateAllUserTables(peerConfig.replicateAllUserTables())
233      .setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap())
234      .setExcludeNamespaces(peerConfig.getExcludeNamespaces())
235      .setBandwidth(peerConfig.getBandwidth()).setSerial(peerConfig.isSerial());
236    return builder;
237  }
238
239  static class ReplicationPeerConfigBuilderImpl implements ReplicationPeerConfigBuilder {
240
241    private String clusterKey;
242
243    private String replicationEndpointImpl;
244
245    private Map<byte[], byte[]> peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR);
246
247    private Map<String, String> configuration = new HashMap<>();
248
249    private Map<TableName, List<String>> tableCFsMap = null;
250
251    private Set<String> namespaces = null;
252
253    // Default value is true, means replicate all user tables to peer cluster.
254    private boolean replicateAllUserTables = true;
255
256    private Map<TableName, List<String>> excludeTableCFsMap = null;
257
258    private Set<String> excludeNamespaces = null;
259
260    private long bandwidth = 0;
261
262    private boolean serial = false;
263
264    @Override
265    public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) {
266      this.clusterKey = clusterKey;
267      return this;
268    }
269
270    @Override
271    public ReplicationPeerConfigBuilder setReplicationEndpointImpl(String replicationEndpointImpl) {
272      this.replicationEndpointImpl = replicationEndpointImpl;
273      return this;
274    }
275
276    @Override
277    public ReplicationPeerConfigBuilder putConfiguration(String key, String value) {
278      this.configuration.put(key, value);
279      return this;
280    }
281
282    @Override
283    public ReplicationPeerConfigBuilder putPeerData(byte[] key, byte[] value) {
284      this.peerData.put(key, value);
285      return this;
286    }
287
288    @Override
289    public ReplicationPeerConfigBuilder
290        setTableCFsMap(Map<TableName, List<String>> tableCFsMap) {
291      this.tableCFsMap = tableCFsMap;
292      return this;
293    }
294
295    @Override
296    public ReplicationPeerConfigBuilder setNamespaces(Set<String> namespaces) {
297      this.namespaces = namespaces;
298      return this;
299    }
300
301    @Override
302    public ReplicationPeerConfigBuilder setReplicateAllUserTables(boolean replicateAllUserTables) {
303      this.replicateAllUserTables = replicateAllUserTables;
304      return this;
305    }
306
307    @Override
308    public ReplicationPeerConfigBuilder
309        setExcludeTableCFsMap(Map<TableName, List<String>> excludeTableCFsMap) {
310      this.excludeTableCFsMap = excludeTableCFsMap;
311      return this;
312    }
313
314    @Override
315    public ReplicationPeerConfigBuilder setExcludeNamespaces(Set<String> excludeNamespaces) {
316      this.excludeNamespaces = excludeNamespaces;
317      return this;
318    }
319
320    @Override
321    public ReplicationPeerConfigBuilder setBandwidth(long bandwidth) {
322      this.bandwidth = bandwidth;
323      return this;
324    }
325
326    @Override
327    public ReplicationPeerConfigBuilder setSerial(boolean serial) {
328      this.serial = serial;
329      return this;
330    }
331
332    @Override
333    public ReplicationPeerConfig build() {
334      // It would be nice to validate the configuration, but we have to work with "old" data
335      // from ZK which makes it much more difficult.
336      return new ReplicationPeerConfig(this);
337    }
338  }
339
340  @Override
341  public String toString() {
342    StringBuilder builder = new StringBuilder("clusterKey=").append(clusterKey).append(",");
343    builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(",");
344    builder.append("replicateAllUserTables=").append(replicateAllUserTables).append(",");
345    if (replicateAllUserTables) {
346      if (excludeNamespaces != null) {
347        builder.append("excludeNamespaces=").append(excludeNamespaces.toString()).append(",");
348      }
349      if (excludeTableCFsMap != null) {
350        builder.append("excludeTableCFsMap=").append(excludeTableCFsMap.toString()).append(",");
351      }
352    } else {
353      if (namespaces != null) {
354        builder.append("namespaces=").append(namespaces.toString()).append(",");
355      }
356      if (tableCFsMap != null) {
357        builder.append("tableCFs=").append(tableCFsMap.toString()).append(",");
358      }
359    }
360    builder.append("bandwidth=").append(bandwidth).append(",");
361    builder.append("serial=").append(serial);
362    return builder.toString();
363  }
364
365  /**
366   * Decide whether the table need replicate to the peer cluster
367   * @param table name of the table
368   * @return true if the table need replicate to the peer cluster
369   */
370  public boolean needToReplicate(TableName table) {
371    return needToReplicate(table, null);
372  }
373
374  /**
375   * Decide whether the passed family of the table need replicate to the peer cluster according to
376   * this peer config.
377   * @param table name of the table
378   * @param family family name
379   * @return true if (the family of) the table need replicate to the peer cluster.
380   *         If passed family is null, return true if any CFs of the table need replicate;
381   *         If passed family is not null, return true if the passed family need replicate.
382   */
383  public boolean needToReplicate(TableName table, byte[] family) {
384    String namespace = table.getNamespaceAsString();
385    if (replicateAllUserTables) {
386      // replicate all user tables, but filter by exclude namespaces and table-cfs config
387      if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
388        return false;
389      }
390      // trap here, must check existence first since HashMap allows null value.
391      if (excludeTableCFsMap == null || !excludeTableCFsMap.containsKey(table)) {
392        return true;
393      }
394      Collection<String> cfs = excludeTableCFsMap.get(table);
395      // If cfs is null or empty then we can make sure that we do not need to replicate this table,
396      // otherwise, we may still need to replicate the table but filter out some families.
397      return cfs != null && !cfs.isEmpty()
398        // If exclude-table-cfs contains passed family then we make sure that we do not need to
399        // replicate this family.
400        && (family == null || !cfs.contains(Bytes.toString(family)));
401    } else {
402      // Not replicate all user tables, so filter by namespaces and table-cfs config
403      if (namespaces == null && tableCFsMap == null) {
404        return false;
405      }
406      // First filter by namespaces config
407      // If table's namespace in peer config, all the tables data are applicable for replication
408      if (namespaces != null && namespaces.contains(namespace)) {
409        return true;
410      }
411      // If table-cfs contains this table then we can make sure that we need replicate some CFs of
412      // this table. Further we need all CFs if tableCFsMap.get(table) is null or empty.
413      return tableCFsMap != null && tableCFsMap.containsKey(table)
414        && (family == null || CollectionUtils.isEmpty(tableCFsMap.get(table))
415        // If table-cfs must contain passed family then we need to replicate this family.
416        || tableCFsMap.get(table).contains(Bytes.toString(family)));
417    }
418  }
419}