1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.rsgroup;
22
23 import com.google.common.collect.Lists;
24 import com.google.common.collect.Maps;
25
26 import com.google.common.collect.Sets;
27 import com.google.protobuf.ServiceException;
28
29 import java.io.IOException;
30 import java.util.ArrayList;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.LinkedList;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.NavigableSet;
39 import java.util.Set;
40 import java.util.TreeSet;
41 import java.util.concurrent.atomic.AtomicBoolean;
42
43 import org.apache.commons.logging.Log;
44 import org.apache.commons.logging.LogFactory;
45 import org.apache.hadoop.hbase.Cell;
46 import org.apache.hadoop.hbase.CellUtil;
47 import org.apache.hadoop.hbase.Coprocessor;
48 import org.apache.hadoop.hbase.DoNotRetryIOException;
49 import org.apache.hadoop.hbase.HColumnDescriptor;
50 import org.apache.hadoop.hbase.HConstants;
51 import org.apache.hadoop.hbase.HRegionInfo;
52 import org.apache.hadoop.hbase.HTableDescriptor;
53 import org.apache.hadoop.hbase.MetaTableAccessor;
54 import org.apache.hadoop.hbase.ServerName;
55 import org.apache.hadoop.hbase.TableName;
56 import org.apache.hadoop.hbase.TableStateManager;
57 import org.apache.hadoop.hbase.client.ClusterConnection;
58 import org.apache.hadoop.hbase.client.Delete;
59 import org.apache.hadoop.hbase.client.Get;
60 import org.apache.hadoop.hbase.client.Mutation;
61 import org.apache.hadoop.hbase.client.Put;
62 import org.apache.hadoop.hbase.client.Result;
63 import org.apache.hadoop.hbase.client.Table;
64 import org.apache.hadoop.hbase.constraint.ConstraintException;
65 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
66 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
67 import org.apache.hadoop.hbase.master.MasterServices;
68 import org.apache.hadoop.hbase.master.ServerListener;
69 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
70 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
71 import org.apache.hadoop.hbase.net.Address;
72 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73 import org.apache.hadoop.hbase.protobuf.RequestConverter;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
76 import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
77 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
78 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
79 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
82 import org.apache.hadoop.hbase.util.Threads;
83 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
84 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
85 import org.apache.zookeeper.KeeperException;
86
87
88
89
90
91
92
93 public class RSGroupInfoManagerImpl implements RSGroupInfoManager, ServerListener {
94 private static final Log LOG = LogFactory.getLog(RSGroupInfoManagerImpl.class);
95
96
97 private final static HTableDescriptor RSGROUP_TABLE_DESC;
98 static {
99 RSGROUP_TABLE_DESC = new HTableDescriptor(RSGROUP_TABLE_NAME);
100 RSGROUP_TABLE_DESC.addFamily(new HColumnDescriptor(META_FAMILY_BYTES));
101 RSGROUP_TABLE_DESC.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName());
102 try {
103 RSGROUP_TABLE_DESC.addCoprocessor(
104 MultiRowMutationEndpoint.class.getName(),
105 null, Coprocessor.PRIORITY_SYSTEM, null);
106 } catch (IOException ex) {
107 throw new RuntimeException(ex);
108 }
109 }
110
111 private volatile Map<String, RSGroupInfo> rsGroupMap;
112 private volatile Map<TableName, String> tableMap;
113 private MasterServices master;
114 private ClusterConnection conn;
115 private ZooKeeperWatcher watcher;
116 private RSGroupStartupWorker rsGroupStartupWorker;
117
118 private volatile Set<String> prevRSGroups;
119 private RSGroupSerDe rsGroupSerDe;
120 private DefaultServerUpdater defaultServerUpdater;
121 private boolean isInit = false;
122
123 public RSGroupInfoManagerImpl(MasterServices master) throws IOException {
124 this.rsGroupMap = Collections.emptyMap();
125 this.tableMap = Collections.emptyMap();
126 rsGroupSerDe = new RSGroupSerDe();
127 this.master = master;
128 this.watcher = master.getZooKeeper();
129 this.conn = master.getConnection();
130 prevRSGroups = new HashSet<String>();
131 }
132
133 public void init() throws IOException{
134 rsGroupStartupWorker = new RSGroupStartupWorker(this, master, conn);
135 refresh();
136 defaultServerUpdater = new DefaultServerUpdater(this);
137 Threads.setDaemonThreadRunning(defaultServerUpdater);
138 master.getServerManager().registerListener(this);
139 isInit = true;
140 }
141
142 boolean isInit() {
143 return isInit;
144 }
145
146 public void start(){
147
148 rsGroupStartupWorker.start();
149 }
150
151
152
153
154
155
156 @Override
157 public synchronized void addRSGroup(RSGroupInfo rsGroupInfo) throws IOException {
158 checkGroupName(rsGroupInfo.getName());
159 if (rsGroupMap.get(rsGroupInfo.getName()) != null ||
160 rsGroupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
161 throw new DoNotRetryIOException("Group already exists: "+ rsGroupInfo.getName());
162 }
163 Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
164 newGroupMap.put(rsGroupInfo.getName(), rsGroupInfo);
165 flushConfig(newGroupMap);
166 }
167
168 @Override
169 public synchronized Set<Address> moveServers(Set<Address> servers, String srcGroup,
170 String dstGroup) throws IOException {
171 if (servers == null) {
172 throw new ConstraintException("The list of servers to move cannot be null.");
173 }
174 Set<Address> movedServers = Sets.newHashSet();
175 if (!rsGroupMap.containsKey(srcGroup)) {
176 throw new DoNotRetryIOException("Group "+srcGroup+" does not exist");
177 }
178 if (!rsGroupMap.containsKey(dstGroup)) {
179 throw new DoNotRetryIOException("Group "+dstGroup+" does not exist");
180 }
181
182 RSGroupInfo src = new RSGroupInfo(getRSGroup(srcGroup));
183 RSGroupInfo dst = new RSGroupInfo(getRSGroup(dstGroup));
184 for(Address el: servers) {
185 if (src.removeServer(el)) {
186 movedServers.add(el);
187 }
188 dst.addServer(el);
189 }
190
191 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
192 newGroupMap.put(src.getName(), src);
193 newGroupMap.put(dst.getName(), dst);
194
195 flushConfig(newGroupMap);
196 return movedServers;
197 }
198
199
200
201
202
203
204
205 @Override
206 public RSGroupInfo getRSGroupOfServer(Address server) throws IOException {
207 for (RSGroupInfo info : rsGroupMap.values()) {
208 if (info.containsServer(server)){
209 return info;
210 }
211 }
212 return null;
213 }
214
215
216
217
218
219
220
221
222 @Override
223 public RSGroupInfo getRSGroup(String groupName) throws IOException {
224 return rsGroupMap.get(groupName);
225 }
226
227
228
229
230
231
232
233 @Override
234 public String getRSGroupOfTable(TableName tableName) throws IOException {
235 return tableMap.get(tableName);
236 }
237
238 @Override
239 public synchronized void moveTables(
240 Set<TableName> tableNames, String groupName) throws IOException {
241
242 if (groupName != null && !rsGroupMap.containsKey(groupName)) {
243 throw new DoNotRetryIOException("Group "+groupName+" does not exist");
244 }
245
246
247 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
248
249
250
251 for(TableName tableName: tableNames) {
252 if (tableMap.containsKey(tableName)) {
253 RSGroupInfo src = new RSGroupInfo(newGroupMap.get(tableMap.get(tableName)));
254 src.removeTable(tableName);
255 newGroupMap.put(src.getName(), src);
256 }
257 }
258
259
260
261 if (groupName != null) {
262 RSGroupInfo dstGroup = new RSGroupInfo(newGroupMap.get(groupName));
263 dstGroup.addAllTables(tableNames);
264 newGroupMap.put(dstGroup.getName(), dstGroup);
265 }
266
267
268 flushConfig(newGroupMap);
269 }
270
271
272
273
274
275
276
277
278 @Override
279 public synchronized void removeRSGroup(String groupName) throws IOException {
280 if (!rsGroupMap.containsKey(groupName) || groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
281 throw new DoNotRetryIOException("Group "+groupName+" does not exist or is a reserved group");
282 }
283 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
284 newGroupMap.remove(groupName);
285 flushConfig(newGroupMap);
286 }
287
288 @Override
289 public List<RSGroupInfo> listRSGroups() throws IOException {
290 List<RSGroupInfo> list = Lists.newLinkedList(rsGroupMap.values());
291 return list;
292 }
293
294 @Override
295 public boolean isOnline() {
296 return rsGroupStartupWorker.isOnline();
297 }
298
299 @Override
300 public synchronized void refresh() throws IOException {
301 refresh(false);
302 }
303
304 private synchronized void refresh(boolean forceOnline) throws IOException {
305 List<RSGroupInfo> groupList = new LinkedList<RSGroupInfo>();
306
307
308
309 if (forceOnline || isOnline()) {
310 LOG.debug("Refreshing in Online mode.");
311 try (Table rsGroupTable = conn.getTable(RSGROUP_TABLE_NAME)) {
312 groupList.addAll(rsGroupSerDe.retrieveGroupList(rsGroupTable));
313 }
314 } else {
315 LOG.debug("Refreshing in Offline mode.");
316 String groupBasePath = ZKUtil.joinZNode(watcher.baseZNode, rsGroupZNode);
317 groupList.addAll(rsGroupSerDe.retrieveGroupList(watcher, groupBasePath));
318 }
319
320
321 NavigableSet<TableName> orphanTables = new TreeSet<TableName>();
322 for(String entry: master.getTableDescriptors().getAll().keySet()) {
323 orphanTables.add(TableName.valueOf(entry));
324 }
325
326 for (RSGroupInfo group: groupList) {
327 if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) {
328 orphanTables.removeAll(group.getTables());
329 }
330 }
331
332
333
334
335 groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP,
336 Sets.newHashSet(getDefaultServers(groupList)),
337 orphanTables));
338
339
340 HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap();
341 HashMap<TableName, String> newTableMap = Maps.newHashMap();
342 for (RSGroupInfo group : groupList) {
343 newGroupMap.put(group.getName(), group);
344 for(TableName table: group.getTables()) {
345 newTableMap.put(table, group.getName());
346 }
347 }
348 rsGroupMap = Collections.unmodifiableMap(newGroupMap);
349 tableMap = Collections.unmodifiableMap(newTableMap);
350
351 prevRSGroups.clear();
352 prevRSGroups.addAll(rsGroupMap.keySet());
353 }
354
355 private synchronized Map<TableName,String> flushConfigTable(Map<String,RSGroupInfo> newGroupMap)
356 throws IOException {
357 Map<TableName,String> newTableMap = Maps.newHashMap();
358 List<Mutation> mutations = Lists.newArrayList();
359
360
361 for(String groupName : prevRSGroups) {
362 if(!newGroupMap.containsKey(groupName)) {
363 Delete d = new Delete(Bytes.toBytes(groupName));
364 mutations.add(d);
365 }
366 }
367
368
369 for(RSGroupInfo RSGroupInfo : newGroupMap.values()) {
370 RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
371 Put p = new Put(Bytes.toBytes(RSGroupInfo.getName()));
372 p.addColumn(META_FAMILY_BYTES,
373 META_QUALIFIER_BYTES,
374 proto.toByteArray());
375 mutations.add(p);
376 for(TableName entry: RSGroupInfo.getTables()) {
377 newTableMap.put(entry, RSGroupInfo.getName());
378 }
379 }
380
381 if(mutations.size() > 0) {
382 multiMutate(mutations);
383 }
384 return newTableMap;
385 }
386
387 private synchronized void flushConfig(Map<String, RSGroupInfo> newGroupMap) throws IOException {
388 Map<TableName, String> newTableMap;
389
390
391
392 if (!isOnline()) {
393 if (newGroupMap == this.rsGroupMap) {
394
395
396 return;
397 }
398
399 Map<String, RSGroupInfo> oldGroupMap = Maps.newHashMap(rsGroupMap);
400 RSGroupInfo oldDefaultGroup = oldGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
401 RSGroupInfo newDefaultGroup = newGroupMap.remove(RSGroupInfo.DEFAULT_GROUP);
402 if (!oldGroupMap.equals(newGroupMap)
403 !oldDefaultGroup.getTables().equals(newDefaultGroup.getTables())
404
405 throw new IOException("Only servers in default group can be updated during offline mode");
406 }
407
408
409 newGroupMap.put(RSGroupInfo.DEFAULT_GROUP, newDefaultGroup);
410
411
412
413 rsGroupMap = newGroupMap;
414
415
416
417
418 return;
419 }
420
421
422 newTableMap = flushConfigTable(newGroupMap);
423
424
425
426 rsGroupMap = Collections.unmodifiableMap(newGroupMap);
427 tableMap = Collections.unmodifiableMap(newTableMap);
428
429
430 try {
431 String groupBasePath = ZKUtil.joinZNode(watcher.baseZNode, rsGroupZNode);
432 ZKUtil.createAndFailSilent(watcher, groupBasePath, ProtobufUtil.PB_MAGIC);
433
434 List<ZKUtil.ZKUtilOp> zkOps = new ArrayList<ZKUtil.ZKUtilOp>(newGroupMap.size());
435 for(String groupName : prevRSGroups) {
436 if(!newGroupMap.containsKey(groupName)) {
437 String znode = ZKUtil.joinZNode(groupBasePath, groupName);
438 zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
439 }
440 }
441
442
443 for(RSGroupInfo RSGroupInfo : newGroupMap.values()) {
444 String znode = ZKUtil.joinZNode(groupBasePath, RSGroupInfo.getName());
445 RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
446 LOG.debug("Updating znode: "+znode);
447 ZKUtil.createAndFailSilent(watcher, znode);
448 zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));
449 zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode,
450 ProtobufUtil.prependPBMagic(proto.toByteArray())));
451 }
452 LOG.debug("Writing ZK GroupInfo count: " + zkOps.size());
453
454 ZKUtil.multiOrSequential(watcher, zkOps, false);
455 } catch (KeeperException e) {
456 LOG.error("Failed to write to rsGroupZNode", e);
457 master.abort("Failed to write to rsGroupZNode", e);
458 throw new IOException("Failed to write to rsGroupZNode",e);
459 }
460
461 prevRSGroups.clear();
462 prevRSGroups.addAll(newGroupMap.keySet());
463 }
464
465 private List<ServerName> getOnlineRS() throws IOException {
466 if (master != null) {
467 return master.getServerManager().getOnlineServersList();
468 }
469 try {
470 LOG.debug("Reading online RS from zookeeper");
471 List<ServerName> servers = new LinkedList<ServerName>();
472 for (String el: ZKUtil.listChildrenNoWatch(watcher, watcher.rsZNode)) {
473 servers.add(ServerName.parseServerName(el));
474 }
475 return servers;
476 } catch (KeeperException e) {
477 throw new IOException("Failed to retrieve server list from zookeeper", e);
478 }
479 }
480
481 private List<Address> getDefaultServers() throws IOException {
482 return getDefaultServers(listRSGroups()
483 }
484
485 private List<Address> getDefaultServers(List<RSGroupInfo> rsGroupInfoList) throws IOException {
486
487 Set<Address> serverAddressesInOtherGroups = new HashSet<>();
488 for (RSGroupInfo group : rsGroupInfoList) {
489 if (!RSGroupInfo.DEFAULT_GROUP.equals(group.getName())) {
490 serverAddressesInOtherGroups.addAll(group.getServers());
491 }
492 }
493
494
495 List<Address> defaultServers = new LinkedList<Address>();
496 for (ServerName serverName : getOnlineRS()) {
497 Address serverAddress = Address.fromParts(serverName.getHostname(), serverName.getPort());
498 if (!serverAddressesInOtherGroups.contains(serverAddress)) {
499 defaultServers.add(serverAddress);
500 }
501 }
502 return defaultServers;
503 }
504
505 private synchronized void updateDefaultServers(
506 Set<Address> server) {
507 RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP);
508 RSGroupInfo newInfo = new RSGroupInfo(info.getName(), server, info.getTables());
509 HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
510 newGroupMap.put(newInfo.getName(), newInfo);
511
512 rsGroupMap = Collections.unmodifiableMap(newGroupMap);
513 }
514
515 @Override
516 public void serverAdded(ServerName serverName) {
517 defaultServerUpdater.serverChanged();
518 }
519
520 @Override
521 public void serverRemoved(ServerName serverName) {
522 defaultServerUpdater.serverChanged();
523 }
524
525 private static class DefaultServerUpdater extends Thread {
526 private static final Log LOG = LogFactory.getLog(DefaultServerUpdater.class);
527 private RSGroupInfoManagerImpl mgr;
528 private volatile boolean hasChanged = false;
529
530 public DefaultServerUpdater(RSGroupInfoManagerImpl mgr) {
531 this.mgr = mgr;
532 setName(DefaultServerUpdater.class.getName()+"-" + mgr.master.getServerName());
533 setDaemon(true);
534 }
535
536 @Override
537 public void run() {
538 List<Address> prevDefaultServers = new LinkedList<Address>();
539 while (!mgr.master.isAborted() && !mgr.master.isStopped()) {
540 try {
541 if (LOG.isDebugEnabled()) {
542 LOG.debug("Updating default servers");
543 }
544 List<Address> servers = mgr.getDefaultServers();
545 Collections.sort(servers, new Comparator<Address>() {
546 @Override
547 public int compare(Address o1, Address o2) {
548 int diff = o1.getHostname().compareTo(o2.getHostname());
549 if (diff != 0) {
550 return diff;
551 }
552 return o1.getPort() - o2.getPort();
553 }
554 });
555 if(!servers.equals(prevDefaultServers)) {
556 mgr.updateDefaultServers(Sets.<Address>newHashSet(servers));
557 prevDefaultServers = servers;
558 LOG.info("Updated with servers: "+servers.size());
559 }
560 try {
561 synchronized (this) {
562 while (!hasChanged) {
563 wait();
564 }
565 hasChanged = false;
566 }
567 } catch (InterruptedException e) {
568 LOG.warn("Interrupted", e);
569 }
570 } catch (IOException e) {
571 LOG.warn("Failed to update default servers", e);
572 }
573 }
574 }
575
576
577 public void serverChanged() {
578 synchronized (this) {
579 hasChanged = true;
580 this.notify();
581 }
582 }
583 }
584
585 @Override
586 public void waiting() {
587
588 }
589
590 private static class RSGroupStartupWorker extends Thread {
591 private static final Log LOG = LogFactory.getLog(RSGroupStartupWorker.class);
592
593 private volatile boolean isOnline = false;
594 private MasterServices masterServices;
595 private RSGroupInfoManagerImpl groupInfoManager;
596 private ClusterConnection conn;
597
598 public RSGroupStartupWorker(RSGroupInfoManagerImpl groupInfoManager,
599 MasterServices masterServices,
600 ClusterConnection conn) {
601 this.masterServices = masterServices;
602 this.groupInfoManager = groupInfoManager;
603 this.conn = conn;
604 setName(RSGroupStartupWorker.class.getName()+"-"+masterServices.getServerName());
605 setDaemon(true);
606 }
607
608 @Override
609 public void run() {
610 if(waitForGroupTableOnline()) {
611 LOG.info("GroupBasedLoadBalancer is now online");
612 }
613 }
614
615 public boolean waitForGroupTableOnline() {
616 final List<HRegionInfo> foundRegions = new LinkedList<HRegionInfo>();
617 final List<HRegionInfo> assignedRegions = new LinkedList<HRegionInfo>();
618 final AtomicBoolean found = new AtomicBoolean(false);
619 final TableStateManager tsm =
620 masterServices.getAssignmentManager().getTableStateManager();
621 boolean createSent = false;
622 while (!found.get() && isMasterRunning()) {
623 foundRegions.clear();
624 assignedRegions.clear();
625 found.set(true);
626 try {
627 boolean rootMetaFound =
628 masterServices.getMetaTableLocator().verifyMetaRegionLocation(
629 conn,
630 masterServices.getZooKeeper(),
631 1);
632 final AtomicBoolean nsFound = new AtomicBoolean(false);
633 if (rootMetaFound) {
634
635 MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
636 @Override
637 public boolean visit(Result row) throws IOException {
638
639 HRegionInfo info = MetaTableAccessor.getHRegionInfo(row);
640 if (info != null) {
641 Cell serverCell =
642 row.getColumnLatestCell(HConstants.CATALOG_FAMILY,
643 HConstants.SERVER_QUALIFIER);
644 if (RSGROUP_TABLE_NAME.equals(info.getTable()) && serverCell != null) {
645 ServerName sn =
646 ServerName.parseVersionedServerName(CellUtil.cloneValue(serverCell));
647 if (sn == null) {
648 found.set(false);
649 } else if (tsm.isTableState(RSGROUP_TABLE_NAME,
650 ZooKeeperProtos.Table.State.ENABLED)) {
651 try {
652 ClientProtos.ClientService.BlockingInterface rs = conn.getClient(sn);
653 ClientProtos.GetRequest request =
654 RequestConverter.buildGetRequest(info.getRegionName(),
655 new Get(ROW_KEY));
656 rs.get(null, request);
657 assignedRegions.add(info);
658 } catch(Exception ex) {
659 LOG.debug("Caught exception while verifying group region", ex);
660 }
661 }
662 foundRegions.add(info);
663 }
664 if (TableName.NAMESPACE_TABLE_NAME.equals(info.getTable())) {
665 Cell cell = row.getColumnLatestCell(HConstants.CATALOG_FAMILY,
666 HConstants.SERVER_QUALIFIER);
667 ServerName sn = null;
668 if(cell != null) {
669 sn = ServerName.parseVersionedServerName(CellUtil.cloneValue(cell));
670 }
671 if (sn == null) {
672 nsFound.set(false);
673 } else if (tsm.isTableState(TableName.NAMESPACE_TABLE_NAME,
674 ZooKeeperProtos.Table.State.ENABLED)) {
675 try {
676 ClientProtos.ClientService.BlockingInterface rs = conn.getClient(sn);
677 ClientProtos.GetRequest request =
678 RequestConverter.buildGetRequest(info.getRegionName(),
679 new Get(ROW_KEY));
680 rs.get(null, request);
681 nsFound.set(true);
682 } catch(Exception ex) {
683 LOG.debug("Caught exception while verifying group region", ex);
684 }
685 }
686 }
687 }
688 return true;
689 }
690 };
691 MetaTableAccessor.fullScan(conn, visitor);
692
693 if (foundRegions.size() < 1 && rootMetaFound && !createSent && nsFound.get()) {
694 groupInfoManager.createGroupTable(masterServices);
695 createSent = true;
696 }
697 LOG.info("Group table: " + RSGROUP_TABLE_NAME + " isOnline: " + found.get()
698 + ", regionCount: " + foundRegions.size() + ", assignCount: "
699 + assignedRegions.size() + ", rootMetaFound: "+rootMetaFound);
700 found.set(found.get() && assignedRegions.size() == foundRegions.size()
701 && foundRegions.size() > 0);
702 } else {
703 LOG.info("Waiting for catalog tables to come online");
704 found.set(false);
705 }
706 if (found.get()) {
707 LOG.debug("With group table online, refreshing cached information.");
708 groupInfoManager.refresh(true);
709 isOnline = true;
710
711 groupInfoManager.flushConfig(groupInfoManager.rsGroupMap);
712 }
713 } catch (RuntimeException e) {
714 throw e;
715 } catch(Exception e) {
716 found.set(false);
717 LOG.warn("Failed to perform check", e);
718 }
719 try {
720 Thread.sleep(100);
721 } catch (InterruptedException e) {
722 LOG.info("Sleep interrupted", e);
723 }
724 }
725 return found.get();
726 }
727
728 public boolean isOnline() {
729 return isOnline;
730 }
731
732 private boolean isMasterRunning() {
733 return !masterServices.isAborted() && !masterServices.isStopped();
734 }
735 }
736
737 private void createGroupTable(MasterServices masterServices) throws IOException {
738 HRegionInfo[] newRegions =
739 ModifyRegionUtils.createHRegionInfos(RSGROUP_TABLE_DESC, null);
740 ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch();
741 masterServices.getMasterProcedureExecutor().submitProcedure(
742 new CreateTableProcedure(
743 masterServices.getMasterProcedureExecutor().getEnvironment(),
744 RSGROUP_TABLE_DESC,
745 newRegions,
746 latch));
747 latch.await();
748
749 int tries = 600;
750 while(masterServices.getAssignmentManager().getRegionStates()
751 .getRegionServerOfRegion(newRegions[0]) == null && tries > 0) {
752 try {
753 Thread.sleep(100);
754 } catch (InterruptedException e) {
755 throw new IOException("Wait interrupted", e);
756 }
757 tries--;
758 }
759 if(tries <= 0) {
760 throw new IOException("Failed to create group table.");
761 }
762 }
763
764 private void multiMutate(List<Mutation> mutations)
765 throws IOException {
766 MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
767 for (Mutation mutation : mutations) {
768 if (mutation instanceof Put) {
769 mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
770 ClientProtos.MutationProto.MutationType.PUT, mutation));
771 } else if (mutation instanceof Delete) {
772 mrmBuilder.addMutationRequest(ProtobufUtil.toMutation(
773 ClientProtos.MutationProto.MutationType.DELETE, mutation));
774 } else {
775 throw new DoNotRetryIOException("multiMutate doesn't support "
776 + mutation.getClass().getName());
777 }
778 }
779 MutateRowsRequest mrm = mrmBuilder.build();
780
781
782 conn.clearRegionCache(RSGROUP_TABLE_NAME);
783 try (Table rsGroupTable = conn.getTable(RSGROUP_TABLE_NAME)) {
784 CoprocessorRpcChannel channel = rsGroupTable.coprocessorService(ROW_KEY);
785 MultiRowMutationProtos.MultiRowMutationService.BlockingInterface service =
786 MultiRowMutationProtos.MultiRowMutationService.newBlockingStub(channel);
787 try {
788 service.mutateRows(null, mrm);
789 } catch (ServiceException ex) {
790 ProtobufUtil.toIOException(ex);
791 }
792 }
793 }
794
795 private void checkGroupName(String groupName) throws ConstraintException {
796 if(!groupName.matches("[a-zA-Z0-9_]+")) {
797 throw new ConstraintException("Group name should only contain alphanumeric characters");
798 }
799 }
800
801 @Override
802 public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String srcGroup,
803 String dstGroup) throws IOException {
804
805 RSGroupInfo srcGroupInfo = getRSGroup(srcGroup);
806 RSGroupInfo dstGroupInfo = getRSGroup(dstGroup);
807
808
809 for (Address el: servers) {
810 srcGroupInfo.removeServer(el);
811 dstGroupInfo.addServer(el);
812 }
813
814 for(TableName tableName: tables) {
815 srcGroupInfo.removeTable(tableName);
816 dstGroupInfo.addTable(tableName);
817 }
818
819
820 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
821 newGroupMap.put(srcGroupInfo.getName(), srcGroupInfo);
822 newGroupMap.put(dstGroupInfo.getName(), dstGroupInfo);
823 flushConfig(newGroupMap);
824 }
825
826 @Override
827 public synchronized void removeServers(Set<Address> servers) throws IOException {
828 Map<String, RSGroupInfo> rsGroupInfos = new HashMap<String, RSGroupInfo>();
829 for (Address el: servers) {
830 RSGroupInfo rsGroupInfo = getRSGroupOfServer(el);
831 if (rsGroupInfo != null) {
832 RSGroupInfo newRsGroupInfo = rsGroupInfos.get(rsGroupInfo.getName());
833 if (newRsGroupInfo == null) {
834 rsGroupInfo.removeServer(el);
835 rsGroupInfos.put(rsGroupInfo.getName(), rsGroupInfo);
836 } else {
837 newRsGroupInfo.removeServer(el);
838 rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo);
839 }
840 } else {
841 LOG.warn("Server " + el + " does not belong to any rsgroup.");
842 }
843 }
844 if (rsGroupInfos.size() > 0) {
845 Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
846 newGroupMap.putAll(rsGroupInfos);
847 flushConfig(newGroupMap);
848 }
849 }
850
851 @Override
852 public void renameRSGroup(String oldName, String newName) throws IOException {
853 checkGroupName(oldName);
854 checkGroupName(newName);
855 if (oldName.equals(RSGroupInfo.DEFAULT_GROUP)) {
856 throw new ConstraintException("Can't rename default rsgroup");
857 }
858 if (rsGroupMap.containsKey(newName)) {
859 throw new ConstraintException("Group already exists: " + newName);
860 }
861
862 RSGroupInfo oldGroup = getRSGroup(oldName);
863 Map<String,RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap);
864 newGroupMap.remove(oldName);
865 RSGroupInfo newGroup = new RSGroupInfo(newName, oldGroup.getServers(), oldGroup.getTables());
866 newGroupMap.put(newName, newGroup);
867 flushConfig(newGroupMap);
868 }
869
870 }