1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.rsgroup;
22
23 import com.google.common.collect.Lists;
24 import com.google.common.collect.Maps;
25
26 import com.google.common.collect.Sets;
27 import com.google.protobuf.ServiceException;
28
29 import java.io.IOException;
30 import java.util.ArrayList;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.LinkedList;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.NavigableSet;
39 import java.util.Set;
40 import java.util.TreeSet;
41 import java.util.concurrent.atomic.AtomicBoolean;
42
43 import org.apache.commons.logging.Log;
44 import org.apache.commons.logging.LogFactory;
45 import org.apache.hadoop.hbase.Cell;
46 import org.apache.hadoop.hbase.CellUtil;
47 import org.apache.hadoop.hbase.Coprocessor;
48 import org.apache.hadoop.hbase.DoNotRetryIOException;
49 import org.apache.hadoop.hbase.HColumnDescriptor;
50 import org.apache.hadoop.hbase.HConstants;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.HTableDescriptor;
53 import org.apache.hadoop.hbase.MetaTableAccessor;
54 import org.apache.hadoop.hbase.ServerName;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.TableStateManager;
57 import org.apache.hadoop.hbase.client.ClusterConnection;
58 import org.apache.hadoop.hbase.client.Delete;
59 import org.apache.hadoop.hbase.client.Get;
60 import org.apache.hadoop.hbase.client.Mutation;
61 import org.apache.hadoop.hbase.client.Put;
62 import org.apache.hadoop.hbase.client.Result;
63 import org.apache.hadoop.hbase.client.Table;
64 import org.apache.hadoop.hbase.constraint.ConstraintException;
65 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
66 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
67 import org.apache.hadoop.hbase.master.MasterServices;
68 import org.apache.hadoop.hbase.master.ServerListener;
69 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
70 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
71 import org.apache.hadoop.hbase.net.Address;
72 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73 import org.apache.hadoop.hbase.protobuf.RequestConverter;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
76 import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
77 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
78 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
79 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
82 import org.apache.hadoop.hbase.util.Threads;
83 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
84 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
85 import org.apache.zookeeper.KeeperException;
86
87
88
89
90
91
92
93 public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListener {
94 private static final Log LOG = LogFactory.getLog(RSGroupInfoManagerImpl.class);
95
96
97 private final static HTableDescriptor RSGROUP_TABLE_DESC;
98 static {
99 RSGROUP_TABLE_DESC = new HTableDescriptor(RSGROUP_TABLE_NAME);
100 RSGROUP_TABLE_DESC.addFamily(new HColumnDescriptor(META_FAMILY_BYTES));
101 RSGROUP_TABLE_DESC.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName());
102 try {
103 RSGROUP_TABLE_DESC.addCoprocessor(
104 MultiRowMutationEndpoint.class.getName(),
105 null, Coprocessor.PRIORITY_SYSTEM, null);
106 } catch (IOException ex) {
107 throw new RuntimeException(ex);
108 }
109 }
110
111 private volatile Map<String, RSGroupInfo> rsGroupMap;
112 private volatile Map<TableName, String> tableMap;
113 private MasterServices master;
114 private ClusterConnection conn;
115 private ZooKeeperWatcher watcher;
116 private RSGroupStartupWorker rsGroupStartupWorker;
117
118 private volatile Set<String> prevRSGroups;
119 private RSGroupSerDe rsGroupSerDe;
120 private DefaultServerUpdater defaultServerUpdater;
121 private boolean isInit = false;
122
123 public RSGroupInfoManagerImpl(MasterServices master) throws IOException {
124 this.rsGroupMap = Collections.emptyMap();
125 this.tableMap = Collections.emptyMap();
126 rsGroupSerDe = new RSGroupSerDe();
127 this.master = master;
128 this.watcher = master.getZooKeeper();
129 this.conn = master.getConnection();
130 prevRSGroups = new HashSet<String>();
131 }
132
133 public void init() throws IOException{
134 rsGroupStartupWorker = new RSGroupStartupWorker(this, master, conn);
135 refresh();
136 defaultServerUpdater = new DefaultServerUpdater(this);
137 Threads.setDaemonThreadRunning(defaultServerUpdater);
138 master.getServerManager().registerListener(this);
139 isInit = true;
140 }
141
142 boolean isInit() {
143 return isInit;
144 }
145
146 public void start(){
147
148 rsGroupStartupWorker.start();
149 }
150
151
152
153
154
155
156 @Override
157 public synchronized void addRSGroup(RSGroupInfo rsGroupInfo) throws IOException {
158 checkGroupName(rsGroupInfo.getName());
159 if (rsGroupMap.get(rsGroupInfo.getName()) != null ||
160 rsGroupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
161 throw new DoNotRetryIOException("Group already exists: "+ rsGroupInfo.getName());
162 }
163 Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
164 newGroupMap.put(rsGroupInfo.getName(), rsGroupInfo);
165 flushConfig(newGroupMap);
166 }
167
168 @Override
169 public synchronized Set<Address> moveServers(Set<Address> servers, String srcGroup,
170 String dstGroup) throws IOException {
171 if (servers == null) {
172 throw new ConstraintException("The list of servers to move cannot be null.");
173 }
174 Set<Address> movedServers = Sets.newHashSet();
175 if (!rsGroupMap.containsKey(srcGroup)) {
176 throw new DoNotRetryIOException("Group "+srcGroup+" does not exist");
177 }
178 if (!rsGroupMap.containsKey(dstGroup)) {
179 throw new DoNotRetryIOException("Group "+dstGroup+" does not exist");
180 }
181
182 RSGroupInfo src = new RSGroupInfo(getRSGroup(srcGroup));
183 RSGroupInfo dst = new RSGroupInfo(getRSGroup(dstGroup));
184 for(Address el: servers) {
185 if (src.removeServer(el)) {
186 movedServers.add(el);
187 }
188 dst.addServer(el);
189 }
190
191 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
192 newGroupMap.put(src.getName(), src);
193 newGroupMap.put(dst.getName(), dst);
194
195 flushConfig(newGroupMap);
196 return movedServers;
197 }
198
199
200
201
202
203
204
205 @Override
206 public RSGroupInfo getRSGroupOfServer(Address server) throws IOException {
207 for (RSGroupInfo info : rsGroupMap.values()) {
208 if (info.containsServer(server)){
209 return info;
210 }
211 }
212 return null;
213 }
214
215
216
217
218
219
220
221
222 @Override
223 public RSGroupInfo getRSGroup(String groupName) throws IOException {
224 return rsGroupMap.get(groupName);
225 }
226
227
228
229
230
231
232
233 @Override
234 public String getRSGroupOfTable(TableName tableName) throws IOException {
235 return tableMap.get(tableName);
236 }
237
238 @Override
239 public synchronized void moveTables(
240 Set<TableName> tableNames, String groupName) throws IOException {
241
242 if (groupName != null && !rsGroupMap.containsKey(groupName)) {
243 throw new DoNotRetryIOException("Group "+groupName+" does not exist");
244 }
245
246
247 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
248
249
250
251 for(TableName tableName: tableNames) {
252 if (tableMap.containsKey(tableName)) {
253 RSGroupInfo src = new RSGroupInfo(newGroupMap.get(tableMap.get(tableName)));
254 src.removeTable(tableName);
255 newGroupMap.put(src.getName(), src);
256 }
257 }
258
259
260
261 if (groupName != null) {
262 RSGroupInfo dstGroup = new RSGroupInfo(newGroupMap.get(groupName));
263 dstGroup.addAllTables(tableNames);
264 newGroupMap.put(dstGroup.getName(), dstGroup);
265 }
266
267
268 flushConfig(newGroupMap);
269 }
270
271
272
273
274
275
276
277
278 @Override
279 public synchronized void removeRSGroup(String groupName) throws IOException {
280 if (!rsGroupMap.containsKey(groupName) || groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
281 throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a reserved group");
282 }
283 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
284 newGroupMap.remove(groupName);
285 flushConfig(newGroupMap);
286 }
287
288 @Override
289 public List<RSGroupInfo> listRSGroups() throws IOException {
290 List<RSGroupInfo> list = Lists.newLinkedList(rsGroupMap.values());
291 return list;
292 }
293
294 @Override
295 public boolean isOnline() {
296 return rsGroupStartupWorker.isOnline();
297 }
298
299 @Override
300 public synchronized void refresh() throws IOException {
301 refresh(false);
302 }
303
304 private synchronized void refresh(boolean forceOnline) throws IOException {
305 List<RSGroupInfo> groupList = new LinkedList<RSGroupInfo>();
306
307
308
309 if (forceOnline || isOnline()) {
310 LOG.debug("Refreshing in Online mode.");
311 try (Table rsGroupTable = conn.getTable(RSGROUP_TABLE_NAME)) {
312 groupList.addAll(rsGroupSerDe.retrieveGroupList(rsGroupTable));
313 }
314 } else {
315 LOG.debug("Refreshing in Offline mode.");
316 String groupBasePath = ZKUtil.joinZNode(watcher.baseZNode, rsGroupZNode);
317 groupList.addAll(rsGroupSerDe.retrieveGroupList(watcher, groupBasePath));
318 }
319
320
321 NavigableSet<TableName> orphanTables = new TreeSet<TableName>();
322 for(String entry: master.getTableDescriptors().getAll().keySet()) {
323 orphanTables.add(TableName.valueOf(entry));
324 }
325
326 for (RSGroupInfo group: groupList) {
327 if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
328 orphanTables.removeAll(group.getTables());
329 }
330 }
331
332
333
334
335 groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP,
336 Sets.newHashSet(getDefaultServers(groupList)),
337 orphanTables));
338
339
340 HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap();
341 HashMap<TableName, String> newTableMap = Maps.newHashMap();
342 for (RSGroupInfo group : groupList) {
343 newGroupMap.put(group.getName(), group);
344 for(TableName table: group.getTables()) {
345 newTableMap.put(table, group.getName());
346 }
347 }
348 rsGroupMap = Collections.unmodifiableMap(newGroupMap);
349 tableMap = Collections.unmodifiableMap(newTableMap);
350
351 prevRSGroups.clear();
352 prevRSGroups.addAll(rsGroupMap.keySet());
353 }
354
355 private synchronized Map<TableName,String> flushConfigTable(Map<String,RSGroupInfo> newGroupMap)
356 throws IOException {
357 Map<TableName,String> newTableMap = Maps.newHashMap();
358 List<Mutation> mutations = Lists.newArrayList();
359
360
361 for(String groupName : prevRSGroups) {
362 if(!newGroupMap.containsKey(groupName)) {
363 Delete d = new Delete(Bytes.toBytes(groupName));
364 mutations.add(d);
365 }
366 }
367
368
369 for(RSGroupInfo RSGroupInfo : newGroupMap.values()) {
370 RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
371 Put p = new Put(Bytes.toBytes(RSGroupInfo.getName()));
372 p.addColumn(META_FAMILY_BYTES,
373 META_QUALIFIER_BYTES,
374 proto.toByteArray());
375 mutations.add(p);
376 for(TableName entry: RSGroupInfo.getTables()) {
377 newTableMap.put(entry, RSGroupInfo.getName());
378 }
379 }
380
381 if(mutations.size() > 0) {
382 multiMutate(mutations);
383 }
384 return newTableMap;
385 }
386
387 private synchronized void flushConfig(Map<String, RSGroupInfo> newGroupMap) throws IOException {
388 Map<TableName, String> newTableMap;
389
390
391
392 if (!isOnline()) {
393 if (newGroupMap == this.rsGroupMap) {
394
395
396 return;
397 }
398
399 Map<String, RSGroupInfo> oldGroupMap = Maps.newHashMap(rsGroupMap);
400 RSGroupInfo oldDefaultGroup = oldGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
401 RSGroupInfo newDefaultGroup = newGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
402 if (!oldGroupMap.equals(newGroupMap)
403 !oldDefaultGroup.getTables().equals(newDefaultGroup.getTables())
404
405 throw new IOException("Only servers in default group can be updated during offline mode");
406 }
407
408
409 newGroupMap.put(RSGroupInfo.DEFAULT_GROUP, newDefaultGroup);
410
411
412
413 rsGroupMap = newGroupMap;
414
415
416
417
418 return;
419 }
420
421
422 newTableMap = flushConfigTable(newGroupMap);
423
424
425
426 rsGroupMap = Collections.unmodifiableMap(newGroupMap);
427 tableMap = Collections.unmodifiableMap(newTableMap);
428
429
430 try {
431 String groupBasePath = ZKUtil.joinZNode(watcher.baseZNode, rsGroupZNode);
432 ZKUtil.createAndFailSilent(watcher, groupBasePath, ProtobufUtil.PB_MAGIC);
433
434 List<ZKUtil.ZKUtilOp> zkOps = new ArrayList<ZKUtil.ZKUtilOp>(newGroupMap.size());
435 for(String groupName : prevRSGroups) {
436 if(!newGroupMap.containsKey(groupName)) {
437 String znode = ZKUtil.joinZNode(groupBasePath, groupName);
438 zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
439 }
440 }
441
442
443 for(RSGroupInfo RSGroupInfo : newGroupMap.values()) {
444 String znode = ZKUtil.joinZNode(groupBasePath, RSGroupInfo.getName());
445 RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
446 LOG.debug("Updating znode: "+znode);
447 ZKUtil.createAndFailSilent(watcher, znode);
448 zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
449 zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
450 ProtobufUtil.prependPBMagic(proto.toByteArray())));
451 }
452 LOG.debug("Writing ZK GroupInfo count: " + zkOps.size());
453
454 ZKUtil.multiOrSequential(watcher, zkOps, false);
455 } catch (KeeperException e) {
456 LOG.error("Failed to write to rsGroupZNode", e);
457 master.abort("Failed to write to rsGroupZNode", e);
458 throw new IOException("Failed to write to rsGroupZNode",e);
459 }
460
461 prevRSGroups.clear();
462 prevRSGroups.addAll(newGroupMap.keySet());
463 }
464
465 private List<ServerName> getOnlineRS() throws IOException {
466 if (master != null) {
467 return master.getServerManager().getOnlineServersList();
468 }
469 try {
470 LOG.debug("Reading online RS from zookeeper");
471 List<ServerName> servers = new LinkedList<ServerName>();
472 for (String el: ZKUtil.listChildrenNoWatch(watcher, watcher.rsZNode)) {
473 servers.add(ServerName.parseServerName(el));
474 }
475 return servers;
476 } catch (KeeperException e) {
477 throw new IOException("Failed to retrieve server list from zookeeper", e);
478 }
479 }
480
481 private List<Address> getDefaultServers() throws IOException {
482 return getDefaultServers(listRSGroups()
483 }
484
485 private List<Address> getDefaultServers(List<RSGroupInfo> rsGroupInfoList) throws IOException {
486
487 Set<Address> serverAddressesInOtherGroups = new HashSet<>();
488 for (RSGroupInfo group : rsGroupInfoList) {
489 if (!RSGroupInfo.DEFAULT_GROUP.equals(group.getName())) {
490 serverAddressesInOtherGroups.addAll(group.getServers());
491 }
492 }
493
494
495 List<Address> defaultServers = new LinkedList<Address>();
496 for (ServerName serverName : getOnlineRS()) {
497 Address serverAddress = Address.fromParts(serverName.getHostname(), serverName.getPort());
498 if (!serverAddressesInOtherGroups.contains(serverAddress)) {
499 defaultServers.add(serverAddress);
500 }
501 }
502 return defaultServers;
503 }
504
505 private synchronized void updateDefaultServers(
506 Set<Address> server) throws IOException {
507 RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
508 RSGroupInfo newInfo = new RSGroupInfo(info.getName(), server, info.getTables());
509 HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
510 newGroupMap.put(newInfo.getName(), newInfo);
511 flushConfig(newGroupMap);
512 }
513
514 @Override
515 public void serverAdded(ServerName serverName) {
516 defaultServerUpdater.serverChanged();
517 }
518
519 @Override
520 public void serverRemoved(ServerName serverName) {
521 defaultServerUpdater.serverChanged();
522 }
523
524 private static class DefaultServerUpdater extends Thread {
525 private static final Log LOG = LogFactory.getLog(DefaultServerUpdater.class);
526 private RSGroupInfoManagerImpl mgr;
527 private volatile boolean hasChanged = false;
528
529 public DefaultServerUpdater(RSGroupInfoManagerImpl mgr) {
530 this.mgr = mgr;
531 setName(DefaultServerUpdater.class.getName()+"-" + mgr.master.getServerName());
532 setDaemon(true);
533 }
534
535 @Override
536 public void run() {
537 List<Address> prevDefaultServers = new LinkedList<Address>();
538 while (!mgr.master.isAborted() && !mgr.master.isStopped()) {
539 try {
540 if (LOG.isDebugEnabled()) {
541 LOG.debug("Updating default servers");
542 }
543 List<Address> servers = mgr.getDefaultServers();
544 Collections.sort(servers, new Comparator<Address>() {
545 @Override
546 public int compare(Address o1, Address o2) {
547 int diff = o1.getHostname().compareTo(o2.getHostname());
548 if (diff != 0) {
549 return diff;
550 }
551 return o1.getPort() - o2.getPort();
552 }
553 });
554 if(!servers.equals(prevDefaultServers)) {
555 mgr.updateDefaultServers(Sets.<Address>newHashSet(servers));
556 prevDefaultServers = servers;
557 LOG.info("Updated with servers: "+servers.size());
558 }
559 try {
560 synchronized (this) {
561 while (!hasChanged) {
562 wait();
563 }
564 hasChanged = false;
565 }
566 } catch (InterruptedException e) {
567 LOG.warn("Interrupted", e);
568 }
569 } catch (IOException e) {
570 LOG.warn("Failed to update default servers", e);
571 }
572 }
573 }
574
575
576 public void serverChanged() {
577 synchronized (this) {
578 hasChanged = true;
579 this.notify();
580 }
581 }
582 }
583
584 @Override
585 public void waiting() {
586
587 }
588
589 private static class RSGroupStartupWorker extends Thread {
590 private static final Log LOG = LogFactory.getLog(RSGroupStartupWorker.class);
591
592 private volatile boolean isOnline = false;
593 private MasterServices masterServices;
594 private RSGroupInfoManagerImpl groupInfoManager;
595 private ClusterConnection conn;
596
597 public RSGroupStartupWorker(RSGroupInfoManagerImpl groupInfoManager,
598 MasterServices masterServices,
599 ClusterConnection conn) {
600 this.masterServices = masterServices;
601 this.groupInfoManager = groupInfoManager;
602 this.conn = conn;
603 setName(RSGroupStartupWorker.class.getName()+"-"+masterServices.getServerName());
604 setDaemon(true);
605 }
606
607 @Override
608 public void run() {
609 if(waitForGroupTableOnline()) {
610 LOG.info("GroupBasedLoadBalancer is now online");
611 }
612 }
613
614 public boolean waitForGroupTableOnline() {
615 final List<HRegionInfo> foundRegions = new LinkedList<HRegionInfo>();
616 final List<HRegionInfo> assignedRegions = new LinkedList<HRegionInfo>();
617 final AtomicBoolean found = new AtomicBoolean(false);
618 final TableStateManager tsm =
619 masterServices.getAssignmentManager().getTableStateManager();
620 boolean createSent = false;
621 while (!found.get() && isMasterRunning()) {
622 foundRegions.clear();
623 assignedRegions.clear();
624 found.set(true);
625 try {
626 boolean rootMetaFound =
627 masterServices.getMetaTableLocator().verifyMetaRegionLocation(
628 conn,
629 masterServices.getZooKeeper(),
630 1);
631 final AtomicBoolean nsFound = new AtomicBoolean(false);
632 if (rootMetaFound) {
633
634 MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
635 @Override
636 public boolean visit(Result row) throws IOException {
637
638 HRegionInfo info = MetaTableAccessor.getHRegionInfo(row);
639 if (info != null) {
640 Cell serverCell =
641 row.getColumnLatestCell(HConstants.CATALOG_FAMILY,
642 HConstants.SERVER_QUALIFIER);
643 if (RSGROUP_TABLE_NAME.equals(info.getTable()) && serverCell != null) {
644 ServerName sn =
645 ServerName.parseVersionedServerName(CellUtil.cloneValue(serverCell));
646 if (sn == null) {
647 found.set(false);
648 } else if (tsm.isTableState(RSGROUP_TABLE_NAME,
649 ZooKeeperProtos.Table.State.ENABLED)) {
650 try {
651 ClientProtos.ClientService.BlockingInterface rs = conn.getClient(sn);
652 ClientProtos.GetRequest request =
653 RequestConverter.buildGetRequest(info.getRegionName(),
654 new Get(ROW_KEY));
655 rs.get(null, request);
656 assignedRegions.add(info);
657 } catch(Exception ex) {
658 LOG.debug("Caught exception while verifying group region", ex);
659 }
660 }
661 foundRegions.add(info);
662 }
663 if (TableName.NAMESPACE_TABLE_NAME.equals(info.getTable())) {
664 Cell cell = row.getColumnLatestCell(HConstants.CATALOG_FAMILY,
665 HConstants.SERVER_QUALIFIER);
666 ServerName sn = null;
667 if(cell != null) {
668 sn = ServerName.parseVersionedServerName(CellUtil.cloneValue(cell));
669 }
670 if (sn == null) {
671 nsFound.set(false);
672 } else if (tsm.isTableState(TableName.NAMESPACE_TABLE_NAME,
673 ZooKeeperProtos.Table.State.ENABLED)) {
674 try {
675 ClientProtos.ClientService.BlockingInterface rs = conn.getClient(sn);
676 ClientProtos.GetRequest request =
677 RequestConverter.buildGetRequest(info.getRegionName(),
678 new Get(ROW_KEY));
679 rs.get(null, request);
680 nsFound.set(true);
681 } catch(Exception ex) {
682 LOG.debug("Caught exception while verifying group region", ex);
683 }
684 }
685 }
686 }
687 return true;
688 }
689 };
690 MetaTableAccessor.fullScan(conn, visitor);
691
692 if (foundRegions.size() < 1 && rootMetaFound && !createSent && nsFound.get()) {
693 groupInfoManager.createGroupTable(masterServices);
694 createSent = true;
695 }
696 LOG.info("Group table: " + RSGROUP_TABLE_NAME + " isOnline: " + found.get()
697 + ", regionCount: " + foundRegions.size() + ", assignCount: "
698 + assignedRegions.size() + ", rootMetaFound: "+rootMetaFound);
699 found.set(found.get() && assignedRegions.size() == foundRegions.size()
700 && foundRegions.size() > 0);
701 } else {
702 LOG.info("Waiting for catalog tables to come online");
703 found.set(false);
704 }
705 if (found.get()) {
706 LOG.debug("With group table online, refreshing cached information.");
707 groupInfoManager.refresh(true);
708 isOnline = true;
709
710 groupInfoManager.flushConfig(groupInfoManager.rsGroupMap);
711 }
712 } catch (RuntimeException e) {
713 throw e;
714 } catch(Exception e) {
715 found.set(false);
716 LOG.warn("Failed to perform check", e);
717 }
718 try {
719 Thread.sleep(100);
720 } catch (InterruptedException e) {
721 LOG.info("Sleep interrupted", e);
722 }
723 }
724 return found.get();
725 }
726
727 public boolean isOnline() {
728 return isOnline;
729 }
730
731 private boolean isMasterRunning() {
732 return !masterServices.isAborted() && !masterServices.isStopped();
733 }
734 }
735
736 private void createGroupTable(MasterServices masterServices) throws IOException {
737 HRegionInfo[] newRegions =
738 ModifyRegionUtils.createHRegionInfos(RSGROUP_TABLE_DESC, null);
739 ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
740 masterServices.getMasterProcedureExecutor().submitProcedure(
741 new CreateTableProcedure(
742 masterServices.getMasterProcedureExecutor().getEnvironment(),
743 RSGROUP_TABLE_DESC,
744 newRegions,
745 latch));
746 latch.await();
747
748 int tries = 600;
749 while(masterServices.getAssignmentManager().getRegionStates()
750 .getRegionServerOfRegion(newRegions[0]) == null && tries > 0) {
751 try {
752 Thread.sleep(100);
753 } catch (InterruptedException e) {
754 throw new IOException("Wait interrupted", e);
755 }
756 tries--;
757 }
758 if(tries <= 0) {
759 throw new IOException("Failed to create group table.");
760 }
761 }
762
763 private void multiMutate(List<Mutation> mutations)
764 throws IOException {
765 MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
766 for (Mutation mutation : mutations) {
767 if (mutation instanceof Put) {
768 mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
769 ClientProtos.MutationProto.MutationType.PUT, mutation));
770 } else if (mutation instanceof Delete) {
771 mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
772 ClientProtos.MutationProto.MutationType.DELETE, mutation));
773 } else {
774 throw new DoNotRetryIOException("multiMutate doesn't support "
775 + mutation.getClass().getName());
776 }
777 }
778 MutateRowsRequest mrm = mrmBuilder.build();
779
780
781 conn.clearRegionCache(RSGROUP_TABLE_NAME);
782 try (Table rsGroupTable = conn.getTable(RSGROUP_TABLE_NAME)) {
783 CoprocessorRpcChannel channel = rsGroupTable.coprocessorService(ROW_KEY);
784 MultiRowMutationProtos.MultiRowMutationService.BlockingInterface service =
785 MultiRowMutationProtos.MultiRowMutationService.newBlockingStub(channel);
786 try {
787 service.mutateRows(null, mrm);
788 } catch (ServiceException ex) {
789 ProtobufUtil.toIOException(ex);
790 }
791 }
792 }
793
794 private void checkGroupName(String groupName) throws ConstraintException {
795 if(!groupName.matches("[a-zA-Z0-9_]+")) {
796 throw new ConstraintException("Group name should only contain alphanumeric characters");
797 }
798 }
799
800 @Override
801 public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String srcGroup,
802 String dstGroup) throws IOException {
803
804 RSGroupInfo srcGroupInfo = getRSGroup(srcGroup);
805 RSGroupInfo dstGroupInfo = getRSGroup(dstGroup);
806
807
808 for (Address el: servers) {
809 srcGroupInfo.removeServer(el);
810 dstGroupInfo.addServer(el);
811 }
812
813 for(TableName tableName: tables) {
814 srcGroupInfo.removeTable(tableName);
815 dstGroupInfo.addTable(tableName);
816 }
817
818
819 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
820 newGroupMap.put(srcGroupInfo.getName(), srcGroupInfo);
821 newGroupMap.put(dstGroupInfo.getName(), dstGroupInfo);
822 flushConfig(newGroupMap);
823 }
824
825 @Override
826 public synchronized void removeServers(Set<Address> servers) throws IOException {
827 Map<String, RSGroupInfo> rsGroupInfos = new HashMap<String, RSGroupInfo>();
828 for (Address el: servers) {
829 RSGroupInfo rsGroupInfo = getRSGroupOfServer(el);
830 if (rsGroupInfo != null) {
831 RSGroupInfo newRsGroupInfo = rsGroupInfos.get(rsGroupInfo.getName());
832 if (newRsGroupInfo == null) {
833 rsGroupInfo.removeServer(el);
834 rsGroupInfos.put(rsGroupInfo.getName(), rsGroupInfo);
835 } else {
836 newRsGroupInfo.removeServer(el);
837 rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo);
838 }
839 } else {
840 LOG.warn("Server " + el + " does not belong to any rsgroup.");
841 }
842 }
843 if (rsGroupInfos.size() > 0) {
844 Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
845 newGroupMap.putAll(rsGroupInfos);
846 flushConfig(newGroupMap);
847 }
848 }
849
850 @Override
851 public void renameRSGroup(String oldName, String newName) throws IOException {
852 checkGroupName(oldName);
853 checkGroupName(newName);
854 if (oldName.equals(RSGroupInfo.DEFAULT_GROUP)) {
855 throw new ConstraintException("Can't rename default rsgroup");
856 }
857 if (rsGroupMap.containsKey(newName)) {
858 throw new ConstraintException("Group already exists: " + newName);
859 }
860
861 RSGroupInfo oldGroup = getRSGroup(oldName);
862 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
863 newGroupMap.remove(oldName);
864 RSGroupInfo newGroup = new RSGroupInfo(newName, oldGroup.getServers(), oldGroup.getTables());
865 newGroupMap.put(newName, newGroup);
866 flushConfig(newGroupMap);
867 }
868
869 }