001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hbase.replication;
020
021import java.util.Collection;
022import java.util.Collections;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.Set;
027import java.util.TreeMap;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.util.Bytes;
030import org.apache.yetus.audience.InterfaceAudience;
031
032import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
033
034/**
035 * A configuration for the replication peer cluster.
036 */
037@InterfaceAudience.Public
038public class ReplicationPeerConfig {
039
040  private String clusterKey;
041  private String replicationEndpointImpl;
042  private final Map<byte[], byte[]> peerData;
043  private final Map<String, String> configuration;
044  private Map<TableName, ? extends Collection<String>> tableCFsMap = null;
045  private Set<String> namespaces = null;
046  // Default value is true, means replicate all user tables to peer cluster.
047  private boolean replicateAllUserTables = true;
048  private Map<TableName, ? extends Collection<String>> excludeTableCFsMap = null;
049  private Set<String> excludeNamespaces = null;
050  private long bandwidth = 0;
051  private final boolean serial;
052
053  private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) {
054    this.clusterKey = builder.clusterKey;
055    this.replicationEndpointImpl = builder.replicationEndpointImpl;
056    this.peerData = Collections.unmodifiableMap(builder.peerData);
057    this.configuration = Collections.unmodifiableMap(builder.configuration);
058    this.tableCFsMap =
059        builder.tableCFsMap != null ? unmodifiableTableCFsMap(builder.tableCFsMap) : null;
060    this.namespaces =
061        builder.namespaces != null ? Collections.unmodifiableSet(builder.namespaces) : null;
062    this.replicateAllUserTables = builder.replicateAllUserTables;
063    this.excludeTableCFsMap =
064        builder.excludeTableCFsMap != null ? unmodifiableTableCFsMap(builder.excludeTableCFsMap)
065            : null;
066    this.excludeNamespaces =
067        builder.excludeNamespaces != null ? Collections.unmodifiableSet(builder.excludeNamespaces)
068            : null;
069    this.bandwidth = builder.bandwidth;
070    this.serial = builder.serial;
071  }
072
073  private Map<TableName, List<String>>
074      unmodifiableTableCFsMap(Map<TableName, List<String>> tableCFsMap) {
075    Map<TableName, List<String>> newTableCFsMap = new HashMap<>();
076    tableCFsMap.forEach((table, cfs) -> newTableCFsMap.put(table,
077      cfs != null ? Collections.unmodifiableList(cfs) : null));
078    return Collections.unmodifiableMap(newTableCFsMap);
079  }
080
081  /**
082   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
083   *             {@link ReplicationPeerConfigBuilder} to create new ReplicationPeerConfig.
084   */
085  @Deprecated
086  public ReplicationPeerConfig() {
087    this.peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR);
088    this.configuration = new HashMap<>(0);
089    this.serial = false;
090  }
091
092  /**
093   * Set the clusterKey which is the concatenation of the slave cluster's:
094   * hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
095   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
096   *             {@link ReplicationPeerConfigBuilder#setClusterKey(String)} instead.
097   */
098  @Deprecated
099  public ReplicationPeerConfig setClusterKey(String clusterKey) {
100    this.clusterKey = clusterKey;
101    return this;
102  }
103
104  /**
105   * Sets the ReplicationEndpoint plugin class for this peer.
106   * @param replicationEndpointImpl a class implementing ReplicationEndpoint
107   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
108   *             {@link ReplicationPeerConfigBuilder#setReplicationEndpointImpl(String)} instead.
109   */
110  @Deprecated
111  public ReplicationPeerConfig setReplicationEndpointImpl(String replicationEndpointImpl) {
112    this.replicationEndpointImpl = replicationEndpointImpl;
113    return this;
114  }
115
116  public String getClusterKey() {
117    return clusterKey;
118  }
119
120  public String getReplicationEndpointImpl() {
121    return replicationEndpointImpl;
122  }
123
124  public Map<byte[], byte[]> getPeerData() {
125    return peerData;
126  }
127
128  public Map<String, String> getConfiguration() {
129    return configuration;
130  }
131
132  public Map<TableName, List<String>> getTableCFsMap() {
133    return (Map<TableName, List<String>>) tableCFsMap;
134  }
135
136  /**
137   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
138   *             {@link ReplicationPeerConfigBuilder#setTableCFsMap(Map)} instead.
139   */
140  @Deprecated
141  public ReplicationPeerConfig setTableCFsMap(Map<TableName,
142                                              ? extends Collection<String>> tableCFsMap) {
143    this.tableCFsMap = tableCFsMap;
144    return this;
145  }
146
147  public Set<String> getNamespaces() {
148    return this.namespaces;
149  }
150
151  /**
152   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
153   *             {@link ReplicationPeerConfigBuilder#setNamespaces(Set)} instead.
154   */
155  @Deprecated
156  public ReplicationPeerConfig setNamespaces(Set<String> namespaces) {
157    this.namespaces = namespaces;
158    return this;
159  }
160
161  public long getBandwidth() {
162    return this.bandwidth;
163  }
164
165  /**
166   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
167   *             {@link ReplicationPeerConfigBuilder#setBandwidth(long)} instead.
168   */
169  @Deprecated
170  public ReplicationPeerConfig setBandwidth(long bandwidth) {
171    this.bandwidth = bandwidth;
172    return this;
173  }
174
175  public boolean replicateAllUserTables() {
176    return this.replicateAllUserTables;
177  }
178
179  /**
180   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
181   *             {@link ReplicationPeerConfigBuilder#setReplicateAllUserTables(boolean)} instead.
182   */
183  @Deprecated
184  public ReplicationPeerConfig setReplicateAllUserTables(boolean replicateAllUserTables) {
185    this.replicateAllUserTables = replicateAllUserTables;
186    return this;
187  }
188
189  public Map<TableName, List<String>> getExcludeTableCFsMap() {
190    return (Map<TableName, List<String>>) excludeTableCFsMap;
191  }
192
193  /**
194   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
195   *             {@link ReplicationPeerConfigBuilder#setExcludeTableCFsMap(Map)} instead.
196   */
197  @Deprecated
198  public ReplicationPeerConfig setExcludeTableCFsMap(Map<TableName,
199                                              ? extends Collection<String>> tableCFsMap) {
200    this.excludeTableCFsMap = tableCFsMap;
201    return this;
202  }
203
204  public Set<String> getExcludeNamespaces() {
205    return this.excludeNamespaces;
206  }
207
208  /**
209   * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use
210   *             {@link ReplicationPeerConfigBuilder#setExcludeNamespaces(Set)} instead.
211   */
212  @Deprecated
213  public ReplicationPeerConfig setExcludeNamespaces(Set<String> namespaces) {
214    this.excludeNamespaces = namespaces;
215    return this;
216  }
217
218  public static ReplicationPeerConfigBuilder newBuilder() {
219    return new ReplicationPeerConfigBuilderImpl();
220  }
221
222  public boolean isSerial() {
223    return serial;
224  }
225
226  public static ReplicationPeerConfigBuilder newBuilder(ReplicationPeerConfig peerConfig) {
227    ReplicationPeerConfigBuilderImpl builder = new ReplicationPeerConfigBuilderImpl();
228    builder.setClusterKey(peerConfig.getClusterKey())
229      .setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl())
230      .putAllPeerData(peerConfig.getPeerData()).putAllConfiguration(peerConfig.getConfiguration())
231      .setTableCFsMap(peerConfig.getTableCFsMap()).setNamespaces(peerConfig.getNamespaces())
232      .setReplicateAllUserTables(peerConfig.replicateAllUserTables())
233      .setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap())
234      .setExcludeNamespaces(peerConfig.getExcludeNamespaces())
235      .setBandwidth(peerConfig.getBandwidth()).setSerial(peerConfig.isSerial());
236    return builder;
237  }
238
239  static class ReplicationPeerConfigBuilderImpl implements ReplicationPeerConfigBuilder {
240
241    private String clusterKey;
242
243    private String replicationEndpointImpl;
244
245    private Map<byte[], byte[]> peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR);
246
247    private Map<String, String> configuration = new HashMap<>();
248
249    private Map<TableName, List<String>> tableCFsMap = null;
250
251    private Set<String> namespaces = null;
252
253    // Default value is true, means replicate all user tables to peer cluster.
254    private boolean replicateAllUserTables = true;
255
256    private Map<TableName, List<String>> excludeTableCFsMap = null;
257
258    private Set<String> excludeNamespaces = null;
259
260    private long bandwidth = 0;
261
262    private boolean serial = false;
263
264    @Override
265    public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) {
266      this.clusterKey = clusterKey;
267      return this;
268    }
269
270    @Override
271    public ReplicationPeerConfigBuilder setReplicationEndpointImpl(String replicationEndpointImpl) {
272      this.replicationEndpointImpl = replicationEndpointImpl;
273      return this;
274    }
275
276    @Override
277    public ReplicationPeerConfigBuilder putConfiguration(String key, String value) {
278      this.configuration.put(key, value);
279      return this;
280    }
281
282    @Override
283    public ReplicationPeerConfigBuilder removeConfiguration(String key) {
284      this.configuration.remove(key);
285      return this;
286    }
287
288    @Override
289    public ReplicationPeerConfigBuilder putPeerData(byte[] key, byte[] value) {
290      this.peerData.put(key, value);
291      return this;
292    }
293
294    @Override
295    public ReplicationPeerConfigBuilder
296        setTableCFsMap(Map<TableName, List<String>> tableCFsMap) {
297      this.tableCFsMap = tableCFsMap;
298      return this;
299    }
300
301    @Override
302    public ReplicationPeerConfigBuilder setNamespaces(Set<String> namespaces) {
303      this.namespaces = namespaces;
304      return this;
305    }
306
307    @Override
308    public ReplicationPeerConfigBuilder setReplicateAllUserTables(boolean replicateAllUserTables) {
309      this.replicateAllUserTables = replicateAllUserTables;
310      return this;
311    }
312
313    @Override
314    public ReplicationPeerConfigBuilder
315        setExcludeTableCFsMap(Map<TableName, List<String>> excludeTableCFsMap) {
316      this.excludeTableCFsMap = excludeTableCFsMap;
317      return this;
318    }
319
320    @Override
321    public ReplicationPeerConfigBuilder setExcludeNamespaces(Set<String> excludeNamespaces) {
322      this.excludeNamespaces = excludeNamespaces;
323      return this;
324    }
325
326    @Override
327    public ReplicationPeerConfigBuilder setBandwidth(long bandwidth) {
328      this.bandwidth = bandwidth;
329      return this;
330    }
331
332    @Override
333    public ReplicationPeerConfigBuilder setSerial(boolean serial) {
334      this.serial = serial;
335      return this;
336    }
337
338    @Override
339    public ReplicationPeerConfig build() {
340      // It would be nice to validate the configuration, but we have to work with "old" data
341      // from ZK which makes it much more difficult.
342      return new ReplicationPeerConfig(this);
343    }
344  }
345
346  @Override
347  public String toString() {
348    StringBuilder builder = new StringBuilder("clusterKey=").append(clusterKey).append(",");
349    builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(",");
350    builder.append("replicateAllUserTables=").append(replicateAllUserTables).append(",");
351    if (replicateAllUserTables) {
352      if (excludeNamespaces != null) {
353        builder.append("excludeNamespaces=").append(excludeNamespaces.toString()).append(",");
354      }
355      if (excludeTableCFsMap != null) {
356        builder.append("excludeTableCFsMap=").append(excludeTableCFsMap.toString()).append(",");
357      }
358    } else {
359      if (namespaces != null) {
360        builder.append("namespaces=").append(namespaces.toString()).append(",");
361      }
362      if (tableCFsMap != null) {
363        builder.append("tableCFs=").append(tableCFsMap.toString()).append(",");
364      }
365    }
366    builder.append("bandwidth=").append(bandwidth).append(",");
367    builder.append("serial=").append(serial);
368    return builder.toString();
369  }
370
371  /**
372   * Decide whether the table need replicate to the peer cluster
373   * @param table name of the table
374   * @return true if the table need replicate to the peer cluster
375   */
376  public boolean needToReplicate(TableName table) {
377    return needToReplicate(table, null);
378  }
379
380  /**
381   * Decide whether the passed family of the table need replicate to the peer cluster according to
382   * this peer config.
383   * @param table name of the table
384   * @param family family name
385   * @return true if (the family of) the table need replicate to the peer cluster.
386   *         If passed family is null, return true if any CFs of the table need replicate;
387   *         If passed family is not null, return true if the passed family need replicate.
388   */
389  public boolean needToReplicate(TableName table, byte[] family) {
390    String namespace = table.getNamespaceAsString();
391    if (replicateAllUserTables) {
392      // replicate all user tables, but filter by exclude namespaces and table-cfs config
393      if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) {
394        return false;
395      }
396      // trap here, must check existence first since HashMap allows null value.
397      if (excludeTableCFsMap == null || !excludeTableCFsMap.containsKey(table)) {
398        return true;
399      }
400      Collection<String> cfs = excludeTableCFsMap.get(table);
401      // If cfs is null or empty then we can make sure that we do not need to replicate this table,
402      // otherwise, we may still need to replicate the table but filter out some families.
403      return cfs != null && !cfs.isEmpty()
404        // If exclude-table-cfs contains passed family then we make sure that we do not need to
405        // replicate this family.
406        && (family == null || !cfs.contains(Bytes.toString(family)));
407    } else {
408      // Not replicate all user tables, so filter by namespaces and table-cfs config
409      if (namespaces == null && tableCFsMap == null) {
410        return false;
411      }
412      // First filter by namespaces config
413      // If table's namespace in peer config, all the tables data are applicable for replication
414      if (namespaces != null && namespaces.contains(namespace)) {
415        return true;
416      }
417      // If table-cfs contains this table then we can make sure that we need replicate some CFs of
418      // this table. Further we need all CFs if tableCFsMap.get(table) is null or empty.
419      return tableCFsMap != null && tableCFsMap.containsKey(table)
420        && (family == null || CollectionUtils.isEmpty(tableCFsMap.get(table))
421        // If table-cfs must contain passed family then we need to replicate this family.
422        || tableCFsMap.get(table).contains(Bytes.toString(family)));
423    }
424  }
425}