001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hbase.replication; 020 021import java.util.Collection; 022import java.util.Collections; 023import java.util.HashMap; 024import java.util.List; 025import java.util.Map; 026import java.util.Set; 027import java.util.TreeMap; 028import org.apache.hadoop.hbase.TableName; 029import org.apache.hadoop.hbase.util.Bytes; 030import org.apache.yetus.audience.InterfaceAudience; 031 032import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils; 033 034/** 035 * A configuration for the replication peer cluster. 036 */ 037@InterfaceAudience.Public 038public class ReplicationPeerConfig { 039 040 private String clusterKey; 041 private String replicationEndpointImpl; 042 private final Map<byte[], byte[]> peerData; 043 private final Map<String, String> configuration; 044 private Map<TableName, ? extends Collection<String>> tableCFsMap = null; 045 private Set<String> namespaces = null; 046 // Default value is true, means replicate all user tables to peer cluster. 047 private boolean replicateAllUserTables = true; 048 private Map<TableName, ? extends Collection<String>> excludeTableCFsMap = null; 049 private Set<String> excludeNamespaces = null; 050 private long bandwidth = 0; 051 private final boolean serial; 052 053 private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) { 054 this.clusterKey = builder.clusterKey; 055 this.replicationEndpointImpl = builder.replicationEndpointImpl; 056 this.peerData = Collections.unmodifiableMap(builder.peerData); 057 this.configuration = Collections.unmodifiableMap(builder.configuration); 058 this.tableCFsMap = 059 builder.tableCFsMap != null ? unmodifiableTableCFsMap(builder.tableCFsMap) : null; 060 this.namespaces = 061 builder.namespaces != null ? Collections.unmodifiableSet(builder.namespaces) : null; 062 this.replicateAllUserTables = builder.replicateAllUserTables; 063 this.excludeTableCFsMap = 064 builder.excludeTableCFsMap != null ? unmodifiableTableCFsMap(builder.excludeTableCFsMap) 065 : null; 066 this.excludeNamespaces = 067 builder.excludeNamespaces != null ? Collections.unmodifiableSet(builder.excludeNamespaces) 068 : null; 069 this.bandwidth = builder.bandwidth; 070 this.serial = builder.serial; 071 } 072 073 private Map<TableName, List<String>> 074 unmodifiableTableCFsMap(Map<TableName, List<String>> tableCFsMap) { 075 Map<TableName, List<String>> newTableCFsMap = new HashMap<>(); 076 tableCFsMap.forEach((table, cfs) -> newTableCFsMap.put(table, 077 cfs != null ? Collections.unmodifiableList(cfs) : null)); 078 return Collections.unmodifiableMap(newTableCFsMap); 079 } 080 081 /** 082 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 083 * {@link ReplicationPeerConfigBuilder} to create new ReplicationPeerConfig. 084 */ 085 @Deprecated 086 public ReplicationPeerConfig() { 087 this.peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR); 088 this.configuration = new HashMap<>(0); 089 this.serial = false; 090 } 091 092 /** 093 * Set the clusterKey which is the concatenation of the slave cluster's: 094 * hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent 095 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 096 * {@link ReplicationPeerConfigBuilder#setClusterKey(String)} instead. 097 */ 098 @Deprecated 099 public ReplicationPeerConfig setClusterKey(String clusterKey) { 100 this.clusterKey = clusterKey; 101 return this; 102 } 103 104 /** 105 * Sets the ReplicationEndpoint plugin class for this peer. 106 * @param replicationEndpointImpl a class implementing ReplicationEndpoint 107 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 108 * {@link ReplicationPeerConfigBuilder#setReplicationEndpointImpl(String)} instead. 109 */ 110 @Deprecated 111 public ReplicationPeerConfig setReplicationEndpointImpl(String replicationEndpointImpl) { 112 this.replicationEndpointImpl = replicationEndpointImpl; 113 return this; 114 } 115 116 public String getClusterKey() { 117 return clusterKey; 118 } 119 120 public String getReplicationEndpointImpl() { 121 return replicationEndpointImpl; 122 } 123 124 public Map<byte[], byte[]> getPeerData() { 125 return peerData; 126 } 127 128 public Map<String, String> getConfiguration() { 129 return configuration; 130 } 131 132 public Map<TableName, List<String>> getTableCFsMap() { 133 return (Map<TableName, List<String>>) tableCFsMap; 134 } 135 136 /** 137 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 138 * {@link ReplicationPeerConfigBuilder#setTableCFsMap(Map)} instead. 139 */ 140 @Deprecated 141 public ReplicationPeerConfig setTableCFsMap(Map<TableName, 142 ? extends Collection<String>> tableCFsMap) { 143 this.tableCFsMap = tableCFsMap; 144 return this; 145 } 146 147 public Set<String> getNamespaces() { 148 return this.namespaces; 149 } 150 151 /** 152 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 153 * {@link ReplicationPeerConfigBuilder#setNamespaces(Set)} instead. 154 */ 155 @Deprecated 156 public ReplicationPeerConfig setNamespaces(Set<String> namespaces) { 157 this.namespaces = namespaces; 158 return this; 159 } 160 161 public long getBandwidth() { 162 return this.bandwidth; 163 } 164 165 /** 166 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 167 * {@link ReplicationPeerConfigBuilder#setBandwidth(long)} instead. 168 */ 169 @Deprecated 170 public ReplicationPeerConfig setBandwidth(long bandwidth) { 171 this.bandwidth = bandwidth; 172 return this; 173 } 174 175 public boolean replicateAllUserTables() { 176 return this.replicateAllUserTables; 177 } 178 179 /** 180 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 181 * {@link ReplicationPeerConfigBuilder#setReplicateAllUserTables(boolean)} instead. 182 */ 183 @Deprecated 184 public ReplicationPeerConfig setReplicateAllUserTables(boolean replicateAllUserTables) { 185 this.replicateAllUserTables = replicateAllUserTables; 186 return this; 187 } 188 189 public Map<TableName, List<String>> getExcludeTableCFsMap() { 190 return (Map<TableName, List<String>>) excludeTableCFsMap; 191 } 192 193 /** 194 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 195 * {@link ReplicationPeerConfigBuilder#setExcludeTableCFsMap(Map)} instead. 196 */ 197 @Deprecated 198 public ReplicationPeerConfig setExcludeTableCFsMap(Map<TableName, 199 ? extends Collection<String>> tableCFsMap) { 200 this.excludeTableCFsMap = tableCFsMap; 201 return this; 202 } 203 204 public Set<String> getExcludeNamespaces() { 205 return this.excludeNamespaces; 206 } 207 208 /** 209 * @deprecated as release of 2.0.0, and it will be removed in 3.0.0. Use 210 * {@link ReplicationPeerConfigBuilder#setExcludeNamespaces(Set)} instead. 211 */ 212 @Deprecated 213 public ReplicationPeerConfig setExcludeNamespaces(Set<String> namespaces) { 214 this.excludeNamespaces = namespaces; 215 return this; 216 } 217 218 public static ReplicationPeerConfigBuilder newBuilder() { 219 return new ReplicationPeerConfigBuilderImpl(); 220 } 221 222 public boolean isSerial() { 223 return serial; 224 } 225 226 public static ReplicationPeerConfigBuilder newBuilder(ReplicationPeerConfig peerConfig) { 227 ReplicationPeerConfigBuilderImpl builder = new ReplicationPeerConfigBuilderImpl(); 228 builder.setClusterKey(peerConfig.getClusterKey()) 229 .setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl()) 230 .putAllPeerData(peerConfig.getPeerData()).putAllConfiguration(peerConfig.getConfiguration()) 231 .setTableCFsMap(peerConfig.getTableCFsMap()).setNamespaces(peerConfig.getNamespaces()) 232 .setReplicateAllUserTables(peerConfig.replicateAllUserTables()) 233 .setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap()) 234 .setExcludeNamespaces(peerConfig.getExcludeNamespaces()) 235 .setBandwidth(peerConfig.getBandwidth()).setSerial(peerConfig.isSerial()); 236 return builder; 237 } 238 239 static class ReplicationPeerConfigBuilderImpl implements ReplicationPeerConfigBuilder { 240 241 private String clusterKey; 242 243 private String replicationEndpointImpl; 244 245 private Map<byte[], byte[]> peerData = new TreeMap<>(Bytes.BYTES_COMPARATOR); 246 247 private Map<String, String> configuration = new HashMap<>(); 248 249 private Map<TableName, List<String>> tableCFsMap = null; 250 251 private Set<String> namespaces = null; 252 253 // Default value is true, means replicate all user tables to peer cluster. 254 private boolean replicateAllUserTables = true; 255 256 private Map<TableName, List<String>> excludeTableCFsMap = null; 257 258 private Set<String> excludeNamespaces = null; 259 260 private long bandwidth = 0; 261 262 private boolean serial = false; 263 264 @Override 265 public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) { 266 this.clusterKey = clusterKey; 267 return this; 268 } 269 270 @Override 271 public ReplicationPeerConfigBuilder setReplicationEndpointImpl(String replicationEndpointImpl) { 272 this.replicationEndpointImpl = replicationEndpointImpl; 273 return this; 274 } 275 276 @Override 277 public ReplicationPeerConfigBuilder putConfiguration(String key, String value) { 278 this.configuration.put(key, value); 279 return this; 280 } 281 282 @Override 283 public ReplicationPeerConfigBuilder putPeerData(byte[] key, byte[] value) { 284 this.peerData.put(key, value); 285 return this; 286 } 287 288 @Override 289 public ReplicationPeerConfigBuilder 290 setTableCFsMap(Map<TableName, List<String>> tableCFsMap) { 291 this.tableCFsMap = tableCFsMap; 292 return this; 293 } 294 295 @Override 296 public ReplicationPeerConfigBuilder setNamespaces(Set<String> namespaces) { 297 this.namespaces = namespaces; 298 return this; 299 } 300 301 @Override 302 public ReplicationPeerConfigBuilder setReplicateAllUserTables(boolean replicateAllUserTables) { 303 this.replicateAllUserTables = replicateAllUserTables; 304 return this; 305 } 306 307 @Override 308 public ReplicationPeerConfigBuilder 309 setExcludeTableCFsMap(Map<TableName, List<String>> excludeTableCFsMap) { 310 this.excludeTableCFsMap = excludeTableCFsMap; 311 return this; 312 } 313 314 @Override 315 public ReplicationPeerConfigBuilder setExcludeNamespaces(Set<String> excludeNamespaces) { 316 this.excludeNamespaces = excludeNamespaces; 317 return this; 318 } 319 320 @Override 321 public ReplicationPeerConfigBuilder setBandwidth(long bandwidth) { 322 this.bandwidth = bandwidth; 323 return this; 324 } 325 326 @Override 327 public ReplicationPeerConfigBuilder setSerial(boolean serial) { 328 this.serial = serial; 329 return this; 330 } 331 332 @Override 333 public ReplicationPeerConfig build() { 334 // It would be nice to validate the configuration, but we have to work with "old" data 335 // from ZK which makes it much more difficult. 336 return new ReplicationPeerConfig(this); 337 } 338 } 339 340 @Override 341 public String toString() { 342 StringBuilder builder = new StringBuilder("clusterKey=").append(clusterKey).append(","); 343 builder.append("replicationEndpointImpl=").append(replicationEndpointImpl).append(","); 344 builder.append("replicateAllUserTables=").append(replicateAllUserTables).append(","); 345 if (replicateAllUserTables) { 346 if (excludeNamespaces != null) { 347 builder.append("excludeNamespaces=").append(excludeNamespaces.toString()).append(","); 348 } 349 if (excludeTableCFsMap != null) { 350 builder.append("excludeTableCFsMap=").append(excludeTableCFsMap.toString()).append(","); 351 } 352 } else { 353 if (namespaces != null) { 354 builder.append("namespaces=").append(namespaces.toString()).append(","); 355 } 356 if (tableCFsMap != null) { 357 builder.append("tableCFs=").append(tableCFsMap.toString()).append(","); 358 } 359 } 360 builder.append("bandwidth=").append(bandwidth).append(","); 361 builder.append("serial=").append(serial); 362 return builder.toString(); 363 } 364 365 /** 366 * Decide whether the table need replicate to the peer cluster 367 * @param table name of the table 368 * @return true if the table need replicate to the peer cluster 369 */ 370 public boolean needToReplicate(TableName table) { 371 return needToReplicate(table, null); 372 } 373 374 /** 375 * Decide whether the passed family of the table need replicate to the peer cluster according to 376 * this peer config. 377 * @param table name of the table 378 * @param family family name 379 * @return true if (the family of) the table need replicate to the peer cluster. 380 * If passed family is null, return true if any CFs of the table need replicate; 381 * If passed family is not null, return true if the passed family need replicate. 382 */ 383 public boolean needToReplicate(TableName table, byte[] family) { 384 String namespace = table.getNamespaceAsString(); 385 if (replicateAllUserTables) { 386 // replicate all user tables, but filter by exclude namespaces and table-cfs config 387 if (excludeNamespaces != null && excludeNamespaces.contains(namespace)) { 388 return false; 389 } 390 // trap here, must check existence first since HashMap allows null value. 391 if (excludeTableCFsMap == null || !excludeTableCFsMap.containsKey(table)) { 392 return true; 393 } 394 Collection<String> cfs = excludeTableCFsMap.get(table); 395 // If cfs is null or empty then we can make sure that we do not need to replicate this table, 396 // otherwise, we may still need to replicate the table but filter out some families. 397 return cfs != null && !cfs.isEmpty() 398 // If exclude-table-cfs contains passed family then we make sure that we do not need to 399 // replicate this family. 400 && (family == null || !cfs.contains(Bytes.toString(family))); 401 } else { 402 // Not replicate all user tables, so filter by namespaces and table-cfs config 403 if (namespaces == null && tableCFsMap == null) { 404 return false; 405 } 406 // First filter by namespaces config 407 // If table's namespace in peer config, all the tables data are applicable for replication 408 if (namespaces != null && namespaces.contains(namespace)) { 409 return true; 410 } 411 // If table-cfs contains this table then we can make sure that we need replicate some CFs of 412 // this table. Further we need all CFs if tableCFsMap.get(table) is null or empty. 413 return tableCFsMap != null && tableCFsMap.containsKey(table) 414 && (family == null || CollectionUtils.isEmpty(tableCFsMap.get(table)) 415 // If table-cfs must contain passed family then we need to replicate this family. 416 || tableCFsMap.get(table).contains(Bytes.toString(family))); 417 } 418 } 419}