1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import com.google.protobuf.ByteString;
22 import com.google.protobuf.ServiceException;
23 import java.io.IOException;
24 import java.net.ConnectException;
25 import java.net.InetAddress;
26 import java.util.ArrayList;
27 import java.util.Collections;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.Iterator;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Map.Entry;
34 import java.util.Set;
35 import java.util.concurrent.ConcurrentHashMap;
36 import java.util.concurrent.ConcurrentNavigableMap;
37 import java.util.concurrent.ConcurrentSkipListMap;
38 import java.util.concurrent.CopyOnWriteArrayList;
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.hbase.ClockOutOfSyncException;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.NotServingRegionException;
46 import org.apache.hadoop.hbase.RegionLoad;
47 import org.apache.hadoop.hbase.Server;
48 import org.apache.hadoop.hbase.ServerLoad;
49 import org.apache.hadoop.hbase.ServerName;
50 import org.apache.hadoop.hbase.YouAreDeadException;
51 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
52 import org.apache.hadoop.hbase.classification.InterfaceAudience;
53 import org.apache.hadoop.hbase.client.ClusterConnection;
54 import org.apache.hadoop.hbase.client.ConnectionFactory;
55 import org.apache.hadoop.hbase.client.RetriesExhaustedException;
56 import org.apache.hadoop.hbase.ipc.FailedServerException;
57 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
58 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
59 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
60 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
61 import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
62 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
64 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
65 import org.apache.hadoop.hbase.protobuf.RequestConverter;
66 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
67 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
68 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
69 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
70 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
71 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds;
72 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.StoreSequenceId;
73 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
74 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
75 import org.apache.hadoop.hbase.regionserver.HRegionServer;
76 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
77 import org.apache.hadoop.hbase.security.User;
78 import org.apache.hadoop.hbase.util.Bytes;
79 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
80 import org.apache.hadoop.hbase.util.RetryCounter;
81 import org.apache.hadoop.hbase.util.RetryCounterFactory;
82 import org.apache.hadoop.hbase.util.Triple;
83 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
84 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
85 import org.apache.zookeeper.KeeperException;
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109 @InterfaceAudience.Private
110 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="JLM_JSR166_UTILCONCURRENT_MONITORENTER",
111 justification="Synchronization on concurrent map is intended")
112 public class ServerManager {
113 public static final String WAIT_ON_REGIONSERVERS_MAXTOSTART =
114 "hbase.master.wait.on.regionservers.maxtostart";
115
116 public static final String WAIT_ON_REGIONSERVERS_MINTOSTART =
117 "hbase.master.wait.on.regionservers.mintostart";
118
119 public static final String WAIT_ON_REGIONSERVERS_TIMEOUT =
120 "hbase.master.wait.on.regionservers.timeout";
121
122 public static final String WAIT_ON_REGIONSERVERS_INTERVAL =
123 "hbase.master.wait.on.regionservers.interval";
124
125 private static final Log LOG = LogFactory.getLog(ServerManager.class);
126
127
128 private volatile boolean clusterShutdown = false;
129
130
131
132
133 private final ConcurrentNavigableMap<byte[], Long> flushedSequenceIdByRegion =
134 new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
135
136
137
138
139 private final ConcurrentNavigableMap<byte[], ConcurrentNavigableMap<byte[], Long>>
140 storeFlushedSequenceIdsByRegion =
141 new ConcurrentSkipListMap<byte[], ConcurrentNavigableMap<byte[], Long>>(Bytes.BYTES_COMPARATOR);
142
143
144 private final ConcurrentNavigableMap<ServerName, ServerLoad> onlineServers =
145 new ConcurrentSkipListMap<ServerName, ServerLoad>();
146
147
148
149
150
151 private final Map<ServerName, AdminService.BlockingInterface> rsAdmins =
152 new HashMap<ServerName, AdminService.BlockingInterface>();
153
154
155
156
157
158 private final ArrayList<ServerName> drainingServers =
159 new ArrayList<ServerName>();
160
161 private final Server master;
162 private final MasterServices services;
163 private final ClusterConnection connection;
164
165 private final DeadServer deadservers = new DeadServer();
166
167 private final long maxSkew;
168 private final long warningSkew;
169
170 private final RetryCounterFactory pingRetryCounterFactory;
171 private final RpcControllerFactory rpcControllerFactory;
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189 private Set<ServerName> queuedDeadServers = new HashSet<ServerName>();
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206 private Map<ServerName, Boolean> requeuedDeadServers
207 = new ConcurrentHashMap<ServerName, Boolean>();
208
209
210 private List<ServerListener> listeners = new CopyOnWriteArrayList<ServerListener>();
211
212
213
214
215
216
217
218 public ServerManager(final Server master, final MasterServices services)
219 throws IOException {
220 this(master, services, true);
221 }
222
223 ServerManager(final Server master, final MasterServices services,
224 final boolean connect) throws IOException {
225 this.master = master;
226 this.services = services;
227 Configuration c = master.getConfiguration();
228 maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
229 warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
230 this.connection = connect ? (ClusterConnection)ConnectionFactory.createConnection(c) : null;
231 int pingMaxAttempts = Math.max(1, master.getConfiguration().getInt(
232 "hbase.master.maximum.ping.server.attempts", 10));
233 int pingSleepInterval = Math.max(1, master.getConfiguration().getInt(
234 "hbase.master.ping.server.retry.sleep.interval", 100));
235 this.pingRetryCounterFactory = new RetryCounterFactory(pingMaxAttempts, pingSleepInterval);
236 this.rpcControllerFactory = this.connection == null
237 ? null
238 : connection.getRpcControllerFactory();
239 }
240
241
242
243
244
245 public void registerListener(final ServerListener listener) {
246 this.listeners.add(listener);
247 }
248
249
250
251
252
253 public boolean unregisterListener(final ServerListener listener) {
254 return this.listeners.remove(listener);
255 }
256
257
258
259
260
261
262
263
264 ServerName regionServerStartup(RegionServerStartupRequest request, InetAddress ia)
265 throws IOException {
266
267
268
269
270
271
272
273
274 final String hostname = request.hasUseThisHostnameInstead() ?
275 request.getUseThisHostnameInstead() :ia.getHostName();
276 ServerName sn = ServerName.valueOf(hostname, request.getPort(),
277 request.getServerStartCode());
278 checkClockSkew(sn, request.getServerCurrentTime());
279 checkIsDead(sn, "STARTUP");
280 if (!checkAndRecordNewServer(sn, ServerLoad.EMPTY_SERVERLOAD)) {
281 LOG.warn("THIS SHOULD NOT HAPPEN, RegionServerStartup"
282 + " could not record the server: " + sn);
283 }
284 return sn;
285 }
286
287 private ConcurrentNavigableMap<byte[], Long> getOrCreateStoreFlushedSequenceId(
288 byte[] regionName) {
289 ConcurrentNavigableMap<byte[], Long> storeFlushedSequenceId =
290 storeFlushedSequenceIdsByRegion.get(regionName);
291 if (storeFlushedSequenceId != null) {
292 return storeFlushedSequenceId;
293 }
294 storeFlushedSequenceId = new ConcurrentSkipListMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
295 ConcurrentNavigableMap<byte[], Long> alreadyPut =
296 storeFlushedSequenceIdsByRegion.putIfAbsent(regionName, storeFlushedSequenceId);
297 return alreadyPut == null ? storeFlushedSequenceId : alreadyPut;
298 }
299
300
301
302
303
304 private void updateLastFlushedSequenceIds(ServerName sn, ServerLoad hsl) {
305 Map<byte[], RegionLoad> regionsLoad = hsl.getRegionsLoad();
306 for (Entry<byte[], RegionLoad> entry : regionsLoad.entrySet()) {
307 byte[] encodedRegionName = Bytes.toBytes(HRegionInfo.encodeRegionName(entry.getKey()));
308 Long existingValue = flushedSequenceIdByRegion.get(encodedRegionName);
309 long l = entry.getValue().getCompleteSequenceId();
310
311 if (LOG.isTraceEnabled()) {
312 LOG.trace(Bytes.toString(encodedRegionName) + ", existingValue=" + existingValue +
313 ", completeSequenceId=" + l);
314 }
315 if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue)) {
316 flushedSequenceIdByRegion.put(encodedRegionName, l);
317 } else if (l != HConstants.NO_SEQNUM && l < existingValue) {
318 LOG.warn("RegionServer " + sn + " indicates a last flushed sequence id ("
319 + l + ") that is less than the previous last flushed sequence id ("
320 + existingValue + ") for region " + Bytes.toString(entry.getKey()) + " Ignoring.");
321 }
322 ConcurrentNavigableMap<byte[], Long> storeFlushedSequenceId =
323 getOrCreateStoreFlushedSequenceId(encodedRegionName);
324 for (StoreSequenceId storeSeqId : entry.getValue().getStoreCompleteSequenceId()) {
325 byte[] family = storeSeqId.getFamilyName().toByteArray();
326 existingValue = storeFlushedSequenceId.get(family);
327 l = storeSeqId.getSequenceId();
328 if (LOG.isTraceEnabled()) {
329 LOG.trace(Bytes.toString(encodedRegionName) + ", family=" + Bytes.toString(family) +
330 ", existingValue=" + existingValue + ", completeSequenceId=" + l);
331 }
332
333 if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue.longValue())) {
334 storeFlushedSequenceId.put(family, l);
335 }
336 }
337 }
338 }
339
340 void regionServerReport(ServerName sn,
341 ServerLoad sl) throws YouAreDeadException {
342 checkIsDead(sn, "REPORT");
343 if (null == this.onlineServers.replace(sn, sl)) {
344
345
346
347
348
349
350 if (!checkAndRecordNewServer(sn, sl)) {
351 LOG.info("RegionServerReport ignored, could not record the server: " + sn);
352 return;
353 }
354 }
355 updateLastFlushedSequenceIds(sn, sl);
356 }
357
358
359
360
361
362
363
364
365
366 boolean checkAndRecordNewServer(
367 final ServerName serverName, final ServerLoad sl) {
368 ServerName existingServer = null;
369 synchronized (this.onlineServers) {
370 existingServer = findServerWithSameHostnamePortWithLock(serverName);
371 if (existingServer != null && (existingServer.getStartcode() > serverName.getStartcode())) {
372 LOG.info("Server serverName=" + serverName + " rejected; we already have "
373 + existingServer.toString() + " registered with same hostname and port");
374 return false;
375 }
376 recordNewServerWithLock(serverName, sl);
377 }
378
379
380 if (!this.listeners.isEmpty()) {
381 for (ServerListener listener : this.listeners) {
382 listener.serverAdded(serverName);
383 }
384 }
385
386
387
388 if (existingServer != null && (existingServer.getStartcode() < serverName.getStartcode())) {
389 LOG.info("Triggering server recovery; existingServer " +
390 existingServer + " looks stale, new server:" + serverName);
391 expireServer(existingServer);
392 }
393 return true;
394 }
395
396
397
398
399
400
401
402
403
404 private void checkClockSkew(final ServerName serverName, final long serverCurrentTime)
405 throws ClockOutOfSyncException {
406 long skew = Math.abs(EnvironmentEdgeManager.currentTime() - serverCurrentTime);
407 if (skew > maxSkew) {
408 String message = "Server " + serverName + " has been " +
409 "rejected; Reported time is too far out of sync with master. " +
410 "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms";
411 LOG.warn(message);
412 throw new ClockOutOfSyncException(message);
413 } else if (skew > warningSkew){
414 String message = "Reported time for server " + serverName + " is out of sync with master " +
415 "by " + skew + "ms. (Warning threshold is " + warningSkew + "ms; " +
416 "error threshold is " + maxSkew + "ms)";
417 LOG.warn(message);
418 }
419 }
420
421
422
423
424
425
426
427
428
429 private void checkIsDead(final ServerName serverName, final String what)
430 throws YouAreDeadException {
431 if (this.deadservers.isDeadServer(serverName)) {
432
433
434 String message = "Server " + what + " rejected; currently processing " +
435 serverName + " as dead server";
436 LOG.debug(message);
437 throw new YouAreDeadException(message);
438 }
439
440
441 if ((this.services == null || ((HMaster) this.services).isInitialized())
442 && this.deadservers.cleanPreviousInstance(serverName)) {
443
444
445 LOG.debug(what + ":" + " Server " + serverName + " came back up," +
446 " removed it from the dead servers list");
447 }
448 }
449
450
451
452
453
454 private ServerName findServerWithSameHostnamePortWithLock(
455 final ServerName serverName) {
456 ServerName end = ServerName.valueOf(serverName.getHostname(), serverName.getPort(),
457 Long.MAX_VALUE);
458
459 ServerName r = onlineServers.lowerKey(end);
460 if (r != null) {
461 if (ServerName.isSameHostnameAndPort(r, serverName)) {
462 return r;
463 }
464 }
465 return null;
466 }
467
468
469
470
471
472
473
474 void recordNewServerWithLock(final ServerName serverName, final ServerLoad sl) {
475 LOG.info("Registering server=" + serverName);
476 this.onlineServers.put(serverName, sl);
477 this.rsAdmins.remove(serverName);
478 }
479
480 public RegionStoreSequenceIds getLastFlushedSequenceId(byte[] encodedRegionName) {
481 RegionStoreSequenceIds.Builder builder = RegionStoreSequenceIds.newBuilder();
482 Long seqId = flushedSequenceIdByRegion.get(encodedRegionName);
483 builder.setLastFlushedSequenceId(seqId != null ? seqId.longValue() : HConstants.NO_SEQNUM);
484 Map<byte[], Long> storeFlushedSequenceId =
485 storeFlushedSequenceIdsByRegion.get(encodedRegionName);
486 if (storeFlushedSequenceId != null) {
487 for (Map.Entry<byte[], Long> entry : storeFlushedSequenceId.entrySet()) {
488 builder.addStoreSequenceId(StoreSequenceId.newBuilder()
489 .setFamilyName(ByteString.copyFrom(entry.getKey()))
490 .setSequenceId(entry.getValue().longValue()).build());
491 }
492 }
493 return builder.build();
494 }
495
496
497
498
499
500 public ServerLoad getLoad(final ServerName serverName) {
501 return this.onlineServers.get(serverName);
502 }
503
504
505
506
507
508
509
510 public double getAverageLoad() {
511 int totalLoad = 0;
512 int numServers = 0;
513 for (ServerLoad sl: this.onlineServers.values()) {
514 numServers++;
515 totalLoad += sl.getNumberOfRegions();
516 }
517 return numServers == 0 ? 0 :
518 (double)totalLoad / (double)numServers;
519 }
520
521
522 public int countOfRegionServers() {
523
524 return this.onlineServers.size();
525 }
526
527
528
529
530 public Map<ServerName, ServerLoad> getOnlineServers() {
531
532 synchronized (this.onlineServers) {
533 return Collections.unmodifiableMap(this.onlineServers);
534 }
535 }
536
537 public DeadServer getDeadServers() {
538 return this.deadservers;
539 }
540
541
542
543
544
545 public boolean areDeadServersInProgress() {
546 return this.deadservers.areDeadServersInProgress();
547 }
548
549 void letRegionServersShutdown() {
550 long previousLogTime = 0;
551 ServerName sn = master.getServerName();
552 ZooKeeperWatcher zkw = master.getZooKeeper();
553 int onlineServersCt;
554 while ((onlineServersCt = onlineServers.size()) > 0){
555
556 if (System.currentTimeMillis() > (previousLogTime + 1000)) {
557 Set<ServerName> remainingServers = onlineServers.keySet();
558 synchronized (onlineServers) {
559 if (remainingServers.size() == 1 && remainingServers.contains(sn)) {
560
561 return;
562 }
563 }
564 StringBuilder sb = new StringBuilder();
565
566 for (ServerName key : remainingServers) {
567 if (sb.length() > 0) {
568 sb.append(", ");
569 }
570 sb.append(key);
571 }
572 LOG.info("Waiting on regionserver(s) to go down " + sb.toString());
573 previousLogTime = System.currentTimeMillis();
574 }
575
576 try {
577 List<String> servers = ZKUtil.listChildrenNoWatch(zkw, zkw.rsZNode);
578 if (servers == null || servers.size() == 0 || (servers.size() == 1
579 && servers.contains(sn.toString()))) {
580 LOG.info("ZK shows there is only the master self online, exiting now");
581
582 break;
583 }
584 } catch (KeeperException ke) {
585 LOG.warn("Failed to list regionservers", ke);
586
587 break;
588 }
589 synchronized (onlineServers) {
590 try {
591 if (onlineServersCt == onlineServers.size()) onlineServers.wait(100);
592 } catch (InterruptedException ignored) {
593
594 }
595 }
596 }
597 }
598
599 private List<String> getRegionServersInZK(final ZooKeeperWatcher zkw)
600 throws KeeperException {
601 return ZKUtil.listChildrenNoWatch(zkw, zkw.rsZNode);
602 }
603
604
605
606
607
608 public synchronized void expireServer(final ServerName serverName) {
609 if (serverName.equals(master.getServerName())) {
610 if (!(master.isAborted() || master.isStopped())) {
611 master.stop("We lost our znode?");
612 }
613 return;
614 }
615 if (!services.isServerCrashProcessingEnabled()) {
616 LOG.info("Master doesn't enable ServerShutdownHandler during initialization, "
617 + "delay expiring server " + serverName);
618 this.queuedDeadServers.add(serverName);
619 return;
620 }
621 if (this.deadservers.isDeadServer(serverName)) {
622
623 LOG.warn("Expiration of " + serverName +
624 " but server shutdown already in progress");
625 return;
626 }
627 moveFromOnlineToDeadServers(serverName);
628
629
630
631 if (this.clusterShutdown) {
632 LOG.info("Cluster shutdown set; " + serverName +
633 " expired; onlineServers=" + this.onlineServers.size());
634 if (this.onlineServers.isEmpty()) {
635 master.stop("Cluster shutdown set; onlineServer=0");
636 }
637 return;
638 }
639
640 boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName) ==
641 AssignmentManager.ServerHostRegion.HOSTING_REGION;
642 ProcedureExecutor<MasterProcedureEnv> procExec = this.services.getMasterProcedureExecutor();
643 procExec.submitProcedure(new ServerCrashProcedure(
644 procExec.getEnvironment(), serverName, true, carryingMeta));
645 LOG.debug("Added=" + serverName +
646 " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta);
647
648
649 if (!this.listeners.isEmpty()) {
650 for (ServerListener listener : this.listeners) {
651 listener.serverRemoved(serverName);
652 }
653 }
654 }
655
656 public void moveFromOnlineToDeadServers(final ServerName sn) {
657 synchronized (onlineServers) {
658 if (!this.onlineServers.containsKey(sn)) {
659 LOG.warn("Expiration of " + sn + " but server not online");
660 }
661
662
663
664 this.deadservers.add(sn);
665 this.onlineServers.remove(sn);
666 onlineServers.notifyAll();
667 }
668 this.rsAdmins.remove(sn);
669 }
670
671 public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitWal) {
672
673
674
675
676
677
678
679
680 if (!services.getAssignmentManager().isFailoverCleanupDone()) {
681 requeuedDeadServers.put(serverName, shouldSplitWal);
682 return;
683 }
684
685 this.deadservers.add(serverName);
686 ProcedureExecutor<MasterProcedureEnv> procExec = this.services.getMasterProcedureExecutor();
687 procExec.submitProcedure(new ServerCrashProcedure(
688 procExec.getEnvironment(), serverName, shouldSplitWal, false));
689 }
690
691
692
693
694
695 synchronized void processQueuedDeadServers() {
696 if (!services.isServerCrashProcessingEnabled()) {
697 LOG.info("Master hasn't enabled ServerShutdownHandler");
698 }
699 Iterator<ServerName> serverIterator = queuedDeadServers.iterator();
700 while (serverIterator.hasNext()) {
701 ServerName tmpServerName = serverIterator.next();
702 expireServer(tmpServerName);
703 serverIterator.remove();
704 requeuedDeadServers.remove(tmpServerName);
705 }
706
707 if (!services.getAssignmentManager().isFailoverCleanupDone()) {
708 LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
709 }
710 for (Map.Entry<ServerName, Boolean> entry : requeuedDeadServers.entrySet()) {
711 processDeadServer(entry.getKey(), entry.getValue());
712 }
713 requeuedDeadServers.clear();
714 }
715
716
717
718
719 public boolean removeServerFromDrainList(final ServerName sn) {
720
721
722
723 if (!this.isServerOnline(sn)) {
724 LOG.warn("Server " + sn + " is not currently online. " +
725 "Removing from draining list anyway, as requested.");
726 }
727
728 return this.drainingServers.remove(sn);
729 }
730
731
732
733
734 public boolean addServerToDrainList(final ServerName sn) {
735
736
737
738 if (!this.isServerOnline(sn)) {
739 LOG.warn("Server " + sn + " is not currently online. " +
740 "Ignoring request to add it to draining list.");
741 return false;
742 }
743
744
745 if (this.drainingServers.contains(sn)) {
746 LOG.warn("Server " + sn + " is already in the draining server list." +
747 "Ignoring request to add it again.");
748 return false;
749 }
750 LOG.info("Server " + sn + " added to draining server list.");
751 return this.drainingServers.add(sn);
752 }
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767 public RegionOpeningState sendRegionOpen(final ServerName server,
768 HRegionInfo region, int versionOfOfflineNode, List<ServerName> favoredNodes)
769 throws IOException {
770 AdminService.BlockingInterface admin = getRsAdmin(server);
771 if (admin == null) {
772 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
773 " failed because no RPC connection found to this server");
774 return RegionOpeningState.FAILED_OPENING;
775 }
776 OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server,
777 region, versionOfOfflineNode, favoredNodes,
778 (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
779 try {
780 OpenRegionResponse response = admin.openRegion(null, request);
781 return ResponseConverter.getRegionOpeningState(response);
782 } catch (ServiceException se) {
783 checkForRSznode(server, se);
784 throw ProtobufUtil.getRemoteException(se);
785 }
786 }
787
788
789
790
791
792
793
794
795
796
797
798
799 private void checkForRSznode(final ServerName serverName, final ServiceException se) {
800 if (se.getCause() == null) return;
801 Throwable t = se.getCause();
802 if (t instanceof ConnectException) {
803
804 } else {
805
806 if (!(t instanceof IOException)) return;
807 if (t.getCause() == null) return;
808 if (!(t.getCause() instanceof FailedServerException)) return;
809
810 }
811 if (!isServerOnline(serverName)) return;
812
813
814
815
816 List<String> servers = null;
817 try {
818 servers = getRegionServersInZK(this.master.getZooKeeper());
819 } catch (KeeperException ke) {
820 LOG.warn("Failed to list regionservers", ke);
821
822 }
823 boolean found = false;
824 if (servers != null) {
825 for (String serverNameAsStr: servers) {
826 ServerName sn = ServerName.valueOf(serverNameAsStr);
827 if (sn.equals(serverName)) {
828
829 found = true;
830 break;
831 }
832 }
833 }
834 if (!found) {
835 LOG.warn("Online server " + serverName.toString() + " has no corresponding " +
836 "ephemeral znode (Did it die before registering in zk?); " +
837 "calling expire to clean it up!");
838 expireServer(serverName);
839 }
840 }
841
842
843
844
845
846
847
848
849
850
851 public List<RegionOpeningState> sendRegionOpen(ServerName server,
852 List<Triple<HRegionInfo, Integer, List<ServerName>>> regionOpenInfos)
853 throws IOException {
854 AdminService.BlockingInterface admin = getRsAdmin(server);
855 if (admin == null) {
856 LOG.warn("Attempting to send OPEN RPC to server " + server.toString() +
857 " failed because no RPC connection found to this server");
858 return null;
859 }
860
861 OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server, regionOpenInfos,
862 (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
863 try {
864 OpenRegionResponse response = admin.openRegion(null, request);
865 return ResponseConverter.getRegionOpeningStateList(response);
866 } catch (ServiceException se) {
867 checkForRSznode(server, se);
868 throw ProtobufUtil.getRemoteException(se);
869 }
870 }
871
872 private HBaseRpcController newRpcController() {
873 return rpcControllerFactory == null ? null : rpcControllerFactory.newController();
874 }
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890 public boolean sendRegionClose(ServerName server, HRegionInfo region,
891 int versionOfClosingNode, ServerName dest, boolean transitionInZK) throws IOException {
892 if (server == null) throw new NullPointerException("Passed server is null");
893 AdminService.BlockingInterface admin = getRsAdmin(server);
894 if (admin == null) {
895 throw new IOException("Attempting to send CLOSE RPC to server " +
896 server.toString() + " for region " +
897 region.getRegionNameAsString() +
898 " failed because no RPC connection found to this server");
899 }
900 HBaseRpcController controller = newRpcController();
901 return ProtobufUtil.closeRegion(controller, admin, server, region.getRegionName(),
902 versionOfClosingNode, dest, transitionInZK);
903 }
904
905 public boolean sendRegionClose(ServerName server,
906 HRegionInfo region, int versionOfClosingNode) throws IOException {
907 return sendRegionClose(server, region, versionOfClosingNode, null, true);
908 }
909
910
911
912
913
914
915
916
917
918 public void sendRegionWarmup(ServerName server,
919 HRegionInfo region) {
920 if (server == null) return;
921 try {
922 AdminService.BlockingInterface admin = getRsAdmin(server);
923 HBaseRpcController controller = newRpcController();
924 ProtobufUtil.warmupRegion(controller, admin, region);
925 } catch (IOException e) {
926 LOG.error("Received exception in RPC for warmup server:" +
927 server + "region: " + region +
928 "exception: " + e);
929 }
930 }
931
932
933
934
935
936 public static void closeRegionSilentlyAndWait(ClusterConnection connection,
937 ServerName server, HRegionInfo region, long timeout) throws IOException, InterruptedException {
938 AdminService.BlockingInterface rs = connection.getAdmin(server);
939 HBaseRpcController controller = connection.getRpcControllerFactory().newController();
940 try {
941 ProtobufUtil.closeRegion(controller, rs, server, region.getRegionName(), false);
942 } catch (IOException e) {
943 LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
944 }
945 long expiration = timeout + System.currentTimeMillis();
946 while (System.currentTimeMillis() < expiration) {
947 controller.reset();
948 try {
949 HRegionInfo rsRegion =
950 ProtobufUtil.getRegionInfo(controller, rs, region.getRegionName());
951 if (rsRegion == null) return;
952 } catch (IOException ioe) {
953 if (ioe instanceof NotServingRegionException)
954 return;
955 LOG.warn("Exception when retrieving regioninfo from: "
956 + region.getRegionNameAsString(), ioe);
957 }
958 Thread.sleep(1000);
959 }
960 throw new IOException("Region " + region + " failed to close within"
961 + " timeout " + timeout);
962 }
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977 public void sendRegionsMerge(ServerName server, HRegionInfo region_a,
978 HRegionInfo region_b, boolean forcible, User user) throws IOException {
979 if (server == null)
980 throw new NullPointerException("Passed server is null");
981 if (region_a == null || region_b == null)
982 throw new NullPointerException("Passed region is null");
983 AdminService.BlockingInterface admin = getRsAdmin(server);
984 if (admin == null) {
985 throw new IOException("Attempting to send MERGE REGIONS RPC to server "
986 + server.toString() + " for region "
987 + region_a.getRegionNameAsString() + ","
988 + region_b.getRegionNameAsString()
989 + " failed because no RPC connection found to this server");
990 }
991 HBaseRpcController controller = newRpcController();
992 ProtobufUtil.mergeRegions(controller, admin, region_a, region_b, forcible, user);
993 }
994
995
996
997
998 public boolean isServerReachable(ServerName server) {
999 if (server == null) throw new NullPointerException("Passed server is null");
1000
1001
1002 RetryCounter retryCounter = pingRetryCounterFactory.create();
1003 while (retryCounter.shouldRetry()) {
1004 synchronized (this.onlineServers) {
1005 if (this.deadservers.isDeadServer(server)) {
1006 return false;
1007 }
1008 }
1009 try {
1010 HBaseRpcController controller = newRpcController();
1011 AdminService.BlockingInterface admin = getRsAdmin(server);
1012 if (admin != null) {
1013 ServerInfo info = ProtobufUtil.getServerInfo(controller, admin);
1014 return info != null && info.hasServerName()
1015 && server.getStartcode() == info.getServerName().getStartCode();
1016 }
1017 } catch (IOException ioe) {
1018 if (LOG.isDebugEnabled()) {
1019 LOG.debug("Couldn't reach " + server + ", try=" + retryCounter.getAttemptTimes() + " of "
1020 + retryCounter.getMaxAttempts(), ioe);
1021 }
1022 try {
1023 retryCounter.sleepUntilNextRetry();
1024 } catch(InterruptedException ie) {
1025 Thread.currentThread().interrupt();
1026 break;
1027 }
1028 }
1029 }
1030 return false;
1031 }
1032
1033
1034
1035
1036
1037
1038
1039 private AdminService.BlockingInterface getRsAdmin(final ServerName sn)
1040 throws IOException {
1041 AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
1042 if (admin == null) {
1043 LOG.debug("New admin connection to " + sn.toString());
1044 if (sn.equals(master.getServerName()) && master instanceof HRegionServer) {
1045
1046 admin = ((HRegionServer)master).getRSRpcServices();
1047 } else {
1048 admin = this.connection.getAdmin(sn);
1049 }
1050 this.rsAdmins.put(sn, admin);
1051 }
1052 return admin;
1053 }
1054
1055
1056
1057
1058
1059
1060 private int getMinToStart() {
1061
1062 int requiredMinToStart = 1;
1063 if (BaseLoadBalancer.tablesOnMaster(master.getConfiguration())) {
1064 if (!BaseLoadBalancer.userTablesOnMaster(master.getConfiguration())) {
1065
1066
1067
1068 requiredMinToStart = 2;
1069 }
1070 }
1071 int minToStart = this.master.getConfiguration().getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
1072
1073 return minToStart == -1 || minToStart < requiredMinToStart? requiredMinToStart: minToStart;
1074 }
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089 public void waitForRegionServers(MonitoredTask status)
1090 throws InterruptedException {
1091 final long interval = this.master.getConfiguration().
1092 getLong(WAIT_ON_REGIONSERVERS_INTERVAL, 1500);
1093 final long timeout = this.master.getConfiguration().
1094 getLong(WAIT_ON_REGIONSERVERS_TIMEOUT, 4500);
1095
1096 int minToStart = getMinToStart();
1097 int maxToStart = this.master.getConfiguration().
1098 getInt(WAIT_ON_REGIONSERVERS_MAXTOSTART, Integer.MAX_VALUE);
1099 if (maxToStart < minToStart) {
1100 LOG.warn(String.format("The value of '%s' (%d) is set less than '%s' (%d), ignoring.",
1101 WAIT_ON_REGIONSERVERS_MAXTOSTART, maxToStart,
1102 WAIT_ON_REGIONSERVERS_MINTOSTART, minToStart));
1103 maxToStart = Integer.MAX_VALUE;
1104 }
1105
1106 long now = System.currentTimeMillis();
1107 final long startTime = now;
1108 long slept = 0;
1109 long lastLogTime = 0;
1110 long lastCountChange = startTime;
1111 int count = countOfRegionServers();
1112 int oldCount = 0;
1113
1114
1115
1116
1117
1118
1119
1120 for (ServerListener listener: this.listeners) {
1121 listener.waiting();
1122 }
1123 while (!this.master.isStopped() && count < maxToStart &&
1124 ((lastCountChange + interval) > now || timeout > slept || count < minToStart)) {
1125
1126 if (oldCount != count || lastLogTime + interval < now) {
1127 lastLogTime = now;
1128 String msg =
1129 "Waiting on RegionServer count=" + count + " to settle; waited="+
1130 slept + "ms, expecting min=" + minToStart + " server(s), max="+ getStrForMax(maxToStart) +
1131 " server(s), " + "timeout=" + timeout + "ms, lastChange=" + (lastCountChange - now) + "ms";
1132 LOG.info(msg);
1133 status.setStatus(msg);
1134 }
1135
1136
1137 final long sleepTime = 50;
1138 Thread.sleep(sleepTime);
1139 now = System.currentTimeMillis();
1140 slept = now - startTime;
1141
1142 oldCount = count;
1143 count = countOfRegionServers();
1144 if (count != oldCount) {
1145 lastCountChange = now;
1146 }
1147 }
1148
1149 LOG.info("Finished wait on RegionServer count=" + count + "; waited=" + slept + "ms," +
1150 " expected min=" + minToStart + " server(s), max=" + getStrForMax(maxToStart) + " server(s),"+
1151 " master is "+ (this.master.isStopped() ? "stopped.": "running")
1152 );
1153 }
1154
1155 private String getStrForMax(final int max) {
1156 return max == Integer.MAX_VALUE? "NO_LIMIT": Integer.toString(max);
1157 }
1158
1159
1160
1161
1162 public List<ServerName> getOnlineServersList() {
1163
1164
1165 return new ArrayList<ServerName>(this.onlineServers.keySet());
1166 }
1167
1168
1169
1170
1171 public List<ServerName> getDrainingServersList() {
1172 return new ArrayList<ServerName>(this.drainingServers);
1173 }
1174
1175
1176
1177
1178 Set<ServerName> getDeadNotExpiredServers() {
1179 return new HashSet<ServerName>(this.queuedDeadServers);
1180 }
1181
1182
1183
1184
1185
1186
1187 void removeRequeuedDeadServers() {
1188 requeuedDeadServers.clear();
1189 }
1190
1191
1192
1193
1194
1195 Map<ServerName, Boolean> getRequeuedDeadServers() {
1196 return Collections.unmodifiableMap(this.requeuedDeadServers);
1197 }
1198
1199 public boolean isServerOnline(ServerName serverName) {
1200 return serverName != null && onlineServers.containsKey(serverName);
1201 }
1202
1203
1204
1205
1206
1207 public boolean isServerWithSameHostnamePortOnline(final ServerName serverName) {
1208 return findServerWithSameHostnamePortWithLock(serverName) != null;
1209 }
1210
1211
1212
1213
1214
1215
1216
1217 public synchronized boolean isServerDead(ServerName serverName) {
1218 return serverName == null || deadservers.isDeadServer(serverName)
1219 || queuedDeadServers.contains(serverName)
1220 || requeuedDeadServers.containsKey(serverName);
1221 }
1222
1223 public void shutdownCluster() {
1224 this.clusterShutdown = true;
1225 this.master.stop("Cluster shutdown requested");
1226 }
1227
1228 public boolean isClusterShutdown() {
1229 return this.clusterShutdown;
1230 }
1231
1232
1233
1234
1235 public void stop() {
1236 if (connection != null) {
1237 try {
1238 connection.close();
1239 } catch (IOException e) {
1240 LOG.error("Attempt to close connection to master failed", e);
1241 }
1242 }
1243 }
1244
1245
1246
1247
1248
1249
1250 public List<ServerName> createDestinationServersList(final List<ServerName> serversToExclude){
1251 final List<ServerName> destServers = getOnlineServersList();
1252
1253 if (serversToExclude != null){
1254 destServers.removeAll(serversToExclude);
1255 }
1256
1257
1258 final List<ServerName> drainingServersCopy = getDrainingServersList();
1259 if (!drainingServersCopy.isEmpty()) {
1260 for (final ServerName server: drainingServersCopy) {
1261 destServers.remove(server);
1262 }
1263 }
1264
1265
1266 removeDeadNotExpiredServers(destServers);
1267 return destServers;
1268 }
1269
1270
1271
1272
1273 public List<ServerName> createDestinationServersList(){
1274 return createDestinationServersList(null);
1275 }
1276
1277
1278
1279
1280
1281
1282
1283 void removeDeadNotExpiredServers(List<ServerName> servers) {
1284 Set<ServerName> deadNotExpiredServersCopy = this.getDeadNotExpiredServers();
1285 if (!deadNotExpiredServersCopy.isEmpty()) {
1286 for (ServerName server : deadNotExpiredServersCopy) {
1287 LOG.debug("Removing dead but not expired server: " + server
1288 + " from eligible server pool.");
1289 servers.remove(server);
1290 }
1291 }
1292 }
1293
1294
1295
1296
1297 void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
1298 for (ServerName serverName : getOnlineServersList()) {
1299 deadservers.cleanAllPreviousInstances(serverName);
1300 }
1301 }
1302
1303
1304
1305
1306 public void removeRegion(final HRegionInfo regionInfo) {
1307 final byte[] encodedName = regionInfo.getEncodedNameAsBytes();
1308 storeFlushedSequenceIdsByRegion.remove(encodedName);
1309 flushedSequenceIdByRegion.remove(encodedName);
1310 }
1311
1312 public boolean isRegionInServerManagerStates(final HRegionInfo hri) {
1313 final byte[] encodedName = hri.getEncodedNameAsBytes();
1314 return (storeFlushedSequenceIdsByRegion.containsKey(encodedName)
1315 || flushedSequenceIdByRegion.containsKey(encodedName));
1316 }
1317
1318
1319
1320
1321 public void removeRegions(final List<HRegionInfo> regions) {
1322 for (HRegionInfo hri: regions) {
1323 removeRegion(hri);
1324 }
1325 }
1326 }