1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import com.google.common.base.Preconditions;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.Comparator;
31 import java.util.SortedSet;
32 import java.util.TreeMap;
33 import java.util.TreeSet;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MetaTableAccessor;
42 import org.apache.hadoop.hbase.RegionTransition;
43 import org.apache.hadoop.hbase.ServerLoad;
44 import org.apache.hadoop.hbase.ServerName;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.hadoop.hbase.TableStateManager;
47 import org.apache.hadoop.hbase.classification.InterfaceAudience;
48 import org.apache.hadoop.hbase.client.Mutation;
49 import org.apache.hadoop.hbase.client.Put;
50 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.master.RegionState.State;
53 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.ConfigUtil;
56 import org.apache.hadoop.hbase.util.FSUtils;
57 import org.apache.hadoop.hbase.util.Pair;
58 import org.apache.hadoop.hbase.util.PairOfSameType;
59 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
60 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
61 import org.apache.zookeeper.KeeperException;
62
63 import com.google.common.annotations.VisibleForTesting;
64 import com.google.common.base.Preconditions;
65
66
67
68
69
70
71
72 @InterfaceAudience.Private
73 public class RegionStates {
74 private static final Log LOG = LogFactory.getLog(RegionStates.class);
75
76 public final static RegionStateStampComparator REGION_STATE_COMPARATOR =
77 new RegionStateStampComparator();
78
79
80
81
82 private static class RegionStateStampComparator implements Comparator<RegionState> {
83 @Override
84 public int compare(RegionState l, RegionState r) {
85 return Long.compare(l.getStamp(), r.getStamp()) == 0 ?
86 Bytes.compareTo(l.getRegion().getRegionName(), r.getRegion().getRegionName()) :
87 Long.compare(l.getStamp(), r.getStamp());
88 }
89 }
90
91
92
93
94 final HashMap<String, RegionState> regionsInTransition =
95 new HashMap<String, RegionState>();
96
97
98
99
100
101 private final Map<String, RegionState> regionStates =
102 new HashMap<String, RegionState>();
103
104
105
106
107 private final Map<TableName, Map<String, RegionState>> regionStatesTableIndex =
108 new HashMap<TableName, Map<String, RegionState>>();
109
110
111
112
113
114 private final Map<ServerName, Set<HRegionInfo>> serverHoldings =
115 new HashMap<ServerName, Set<HRegionInfo>>();
116
117
118
119
120 private final Map<HRegionInfo, Set<HRegionInfo>> defaultReplicaToOtherReplicas =
121 new HashMap<HRegionInfo, Set<HRegionInfo>>();
122
123
124
125
126
127 private final TreeMap<HRegionInfo, ServerName> regionAssignments =
128 new TreeMap<HRegionInfo, ServerName>();
129
130
131
132
133
134
135
136
137
138
139
140 private final HashMap<String, ServerName> lastAssignments =
141 new HashMap<String, ServerName>();
142
143
144
145
146
147
148
149
150
151
152 private final HashMap<String, ServerName> oldAssignments =
153 new HashMap<String, ServerName>();
154
155
156
157
158
159
160 private final HashMap<String, Long> deadServers =
161 new HashMap<String, Long>();
162
163
164
165
166
167
168
169
170 private final HashMap<ServerName, Long> processedServers =
171 new HashMap<ServerName, Long>();
172 private long lastProcessedServerCleanTime;
173
174 private final TableStateManager tableStateManager;
175 private final RegionStateStore regionStateStore;
176 private final ServerManager serverManager;
177 private final MasterServices server;
178 private final boolean useZK;
179
180
181 static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime";
182 static final long DEFAULT_LOG_SPLIT_TIME = 7200000L;
183
184 RegionStates(final MasterServices master, final TableStateManager tableStateManager,
185 final ServerManager serverManager, final RegionStateStore regionStateStore) {
186 this.tableStateManager = tableStateManager;
187 this.regionStateStore = regionStateStore;
188 this.serverManager = serverManager;
189 this.server = master;
190 this.useZK = ConfigUtil.useZKForAssignment(server.getConfiguration());
191 }
192
193
194
195
196 public synchronized Map<HRegionInfo, ServerName> getRegionAssignments() {
197 return new TreeMap<HRegionInfo, ServerName>(regionAssignments);
198 }
199
200
201
202
203
204
205 synchronized Map<ServerName, List<HRegionInfo>> getRegionAssignments(
206 Collection<HRegionInfo> regions) {
207 Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
208 for (HRegionInfo region : regions) {
209 HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(region);
210 Set<HRegionInfo> allReplicas = defaultReplicaToOtherReplicas.get(defaultReplica);
211 if (allReplicas != null) {
212 for (HRegionInfo hri : allReplicas) {
213 ServerName server = regionAssignments.get(hri);
214 if (server != null) {
215 List<HRegionInfo> regionsOnServer = map.get(server);
216 if (regionsOnServer == null) {
217 regionsOnServer = new ArrayList<HRegionInfo>(1);
218 map.put(server, regionsOnServer);
219 }
220 regionsOnServer.add(hri);
221 }
222 }
223 }
224 }
225 return map;
226 }
227
228 public synchronized ServerName getRegionServerOfRegion(HRegionInfo hri) {
229 return regionAssignments.get(hri);
230 }
231
232
233
234
235 public synchronized Set<RegionState> getRegionsInTransition() {
236 return new HashSet<RegionState>(regionsInTransition.values());
237 }
238
239
240
241
242 public synchronized Set<RegionState> getAllRegions() {
243 return new HashSet<RegionState>(regionStates.values());
244 }
245
246
247
248
249 public synchronized SortedSet<RegionState> getRegionsInTransitionOrderedByTimestamp() {
250 final TreeSet<RegionState> rit = new TreeSet<RegionState>(REGION_STATE_COMPARATOR);
251 for (RegionState rs: regionsInTransition.values()) {
252 rit.add(rs);
253 }
254 return rit;
255 }
256
257
258
259
260 public synchronized int getRegionsInTransitionCount() {
261 return regionsInTransition.size();
262 }
263
264
265
266
267 public synchronized boolean isRegionInTransition(final HRegionInfo hri) {
268 return regionsInTransition.containsKey(hri.getEncodedName());
269 }
270
271
272
273
274 public synchronized boolean isRegionInTransition(final String encodedName) {
275 return regionsInTransition.containsKey(encodedName);
276 }
277
278
279
280
281 public synchronized boolean isRegionsInTransition() {
282 return !regionsInTransition.isEmpty();
283 }
284
285
286
287
288 public synchronized boolean isMetaRegionInTransition() {
289 for (RegionState state : regionsInTransition.values()) {
290 if (state.getRegion().isMetaRegion()) return true;
291 }
292 return false;
293 }
294
295
296
297
298 public synchronized boolean isRegionOnline(final HRegionInfo hri) {
299 return !isRegionInTransition(hri) && regionAssignments.containsKey(hri);
300 }
301
302
303
304
305
306 public synchronized boolean isRegionOffline(final HRegionInfo hri) {
307 return getRegionState(hri) == null || (!isRegionInTransition(hri)
308 && isRegionInState(hri, State.OFFLINE, State.CLOSED));
309 }
310
311
312
313
314 public boolean isRegionInState(
315 final HRegionInfo hri, final State... states) {
316 return isRegionInState(hri.getEncodedName(), states);
317 }
318
319
320
321
322 public boolean isRegionInState(
323 final String encodedName, final State... states) {
324 RegionState regionState = getRegionState(encodedName);
325 return isOneOfStates(regionState, states);
326 }
327
328
329
330
331 public synchronized void waitForUpdate(
332 final long timeout) throws InterruptedException {
333 this.wait(timeout);
334 }
335
336
337
338
339 public RegionState getRegionTransitionState(final HRegionInfo hri) {
340 return getRegionTransitionState(hri.getEncodedName());
341 }
342
343
344
345
346 public synchronized RegionState
347 getRegionTransitionState(final String encodedName) {
348 return regionsInTransition.get(encodedName);
349 }
350
351
352
353
354
355
356 public void createRegionStates(
357 final List<HRegionInfo> hris) {
358 for (HRegionInfo hri: hris) {
359 createRegionState(hri);
360 }
361 }
362
363
364
365
366
367
368
369 public RegionState createRegionState(final HRegionInfo hri) {
370 return createRegionState(hri, null, null, null);
371 }
372
373
374
375
376
377
378
379
380
381
382
383
384 public synchronized RegionState createRegionState(final HRegionInfo hri,
385 State newState, ServerName serverName, ServerName lastHost) {
386 if (newState == null || (newState == State.OPEN && serverName == null)) {
387 newState = State.OFFLINE;
388 }
389 if (hri.isOffline() && hri.isSplit()) {
390 newState = State.SPLIT;
391 serverName = null;
392 }
393 String encodedName = hri.getEncodedName();
394 RegionState regionState = regionStates.get(encodedName);
395 if (regionState != null) {
396 LOG.warn("Tried to create a state for a region already in RegionStates, "
397 + "used existing: " + regionState + ", ignored new: " + newState);
398 } else {
399 regionState = new RegionState(hri, newState, serverName);
400 putRegionState(regionState);
401 if (newState == State.OPEN) {
402 if (!serverName.equals(lastHost)) {
403 LOG.warn("Open region's last host " + lastHost
404 + " should be the same as the current one " + serverName
405 + ", ignored the last and used the current one");
406 lastHost = serverName;
407 }
408 lastAssignments.put(encodedName, lastHost);
409 regionAssignments.put(hri, lastHost);
410 } else if (!regionState.isUnassignable()) {
411 regionsInTransition.put(encodedName, regionState);
412 }
413 if (lastHost != null && newState != State.SPLIT) {
414 addToReplicaMapping(hri);
415 addToServerHoldings(lastHost, hri);
416 if (newState != State.OPEN) {
417 oldAssignments.put(encodedName, lastHost);
418 }
419 }
420 }
421 return regionState;
422 }
423
424 private RegionState putRegionState(RegionState regionState) {
425 HRegionInfo hri = regionState.getRegion();
426 String encodedName = hri.getEncodedName();
427 TableName table = hri.getTable();
428 RegionState oldState = regionStates.put(encodedName, regionState);
429 Map<String, RegionState> map = regionStatesTableIndex.get(table);
430 if (map == null) {
431 map = new HashMap<String, RegionState>();
432 regionStatesTableIndex.put(table, map);
433 }
434 map.put(encodedName, regionState);
435 return oldState;
436 }
437
438
439
440
441 public RegionState setRegionStateTOCLOSED(
442 final byte[] regionName,
443 final ServerName serverName) {
444 HRegionInfo regionInfo = getRegionInfo(regionName);
445 return setRegionStateTOCLOSED(regionInfo, serverName);
446 }
447
448
449
450
451 public RegionState setRegionStateTOCLOSED(
452 final HRegionInfo regionInfo,
453 final ServerName serverName) {
454 ServerName sn = serverName;
455 if (sn == null) {
456 RegionState regionState = getRegionState(regionInfo.getEncodedName());
457 if (regionState != null) {
458 sn = regionState.getServerName();
459 }
460
461
462
463 }
464
465
466
467
468
469 setLastRegionServerOfRegion(sn, regionInfo.getEncodedName());
470 return updateRegionState(regionInfo, State.CLOSED, sn);
471 }
472
473
474
475
476 public RegionState updateRegionState(
477 final HRegionInfo hri, final State state) {
478 RegionState regionState = getRegionState(hri.getEncodedName());
479 return updateRegionState(hri, state,
480 regionState == null ? null : regionState.getServerName());
481 }
482
483
484
485
486
487
488
489 public RegionState updateRegionState(
490 final RegionTransition transition, final State state) {
491 byte [] regionName = transition.getRegionName();
492 HRegionInfo regionInfo = getRegionInfo(regionName);
493 if (regionInfo == null) {
494 String prettyRegionName = HRegionInfo.prettyPrint(
495 HRegionInfo.encodeRegionName(regionName));
496 LOG.warn("Failed to find region " + prettyRegionName
497 + " in updating its state to " + state
498 + " based on region transition " + transition);
499 return null;
500 }
501 return updateRegionState(regionInfo, state,
502 transition.getServerName());
503 }
504
505
506
507
508 public synchronized RegionState transitionOpenFromPendingOpenOrOpeningOnServer(
509 final RegionTransition transition, final RegionState fromState, final ServerName sn) {
510 if(fromState.isPendingOpenOrOpeningOnServer(sn)){
511 return updateRegionState(transition, State.OPEN);
512 }
513 return null;
514 }
515
516
517
518
519 public RegionState updateRegionState(
520 final HRegionInfo hri, final State state, final ServerName serverName) {
521 return updateRegionState(hri, state, serverName, HConstants.NO_SEQNUM);
522 }
523
524 public void regionOnline(final HRegionInfo hri, final ServerName serverName) {
525 regionOnline(hri, serverName, HConstants.NO_SEQNUM);
526 }
527
528
529
530
531
532
533 public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) {
534 String encodedName = hri.getEncodedName();
535 if (!serverManager.isServerOnline(serverName)) {
536
537
538
539
540 LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName);
541 return;
542 }
543 updateRegionState(hri, State.OPEN, serverName, openSeqNum);
544
545 synchronized (this) {
546 RegionState regionState = regionsInTransition.remove(encodedName);
547
548
549 if (regionState != null && this.server.getAssignmentManager() != null) {
550 long ritDuration = System.currentTimeMillis() - regionState.getStamp()
551 + regionState.getRitDuration();
552 this.server.getAssignmentManager().getAssignmentManagerMetrics()
553 .updateRitDuration(ritDuration);
554 }
555 ServerName oldServerName = regionAssignments.put(hri, serverName);
556 if (!serverName.equals(oldServerName)) {
557 if (LOG.isDebugEnabled()) {
558 LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName);
559 }
560 addToServerHoldings(serverName, hri);
561 addToReplicaMapping(hri);
562 if (oldServerName == null) {
563 oldServerName = oldAssignments.remove(encodedName);
564 }
565 if (oldServerName != null
566 && !oldServerName.equals(serverName)
567 && serverHoldings.containsKey(oldServerName)) {
568 LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
569 removeFromServerHoldings(oldServerName, hri);
570 }
571 }
572 }
573 }
574
575 private void addToServerHoldings(ServerName serverName, HRegionInfo hri) {
576 Set<HRegionInfo> regions = serverHoldings.get(serverName);
577 if (regions == null) {
578 regions = new HashSet<HRegionInfo>();
579 serverHoldings.put(serverName, regions);
580 }
581 regions.add(hri);
582 }
583
584 private void addToReplicaMapping(HRegionInfo hri) {
585 HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
586 Set<HRegionInfo> replicas =
587 defaultReplicaToOtherReplicas.get(defaultReplica);
588 if (replicas == null) {
589 replicas = new HashSet<HRegionInfo>();
590 defaultReplicaToOtherReplicas.put(defaultReplica, replicas);
591 }
592 replicas.add(hri);
593 }
594
595 private void removeFromServerHoldings(ServerName serverName, HRegionInfo hri) {
596 Set<HRegionInfo> oldRegions = serverHoldings.get(serverName);
597 oldRegions.remove(hri);
598 if (oldRegions.isEmpty()) {
599 serverHoldings.remove(serverName);
600 }
601 }
602
603 private void removeFromReplicaMapping(HRegionInfo hri) {
604 HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
605 Set<HRegionInfo> replicas = defaultReplicaToOtherReplicas.get(defaultReplica);
606 if (replicas != null) {
607 replicas.remove(hri);
608 if (replicas.isEmpty()) {
609 defaultReplicaToOtherReplicas.remove(defaultReplica);
610 }
611 }
612 }
613
614
615
616
617 synchronized boolean existsInServerHoldings(final ServerName serverName,
618 final HRegionInfo hri) {
619 Set<HRegionInfo> oldRegions = serverHoldings.get(serverName);
620 if (oldRegions != null) {
621 return oldRegions.contains(hri);
622 }
623 return false;
624 }
625
626
627
628
629
630 public synchronized void logSplit(final ServerName serverName) {
631 for (Iterator<Map.Entry<String, ServerName>> it
632 = lastAssignments.entrySet().iterator(); it.hasNext();) {
633 Map.Entry<String, ServerName> e = it.next();
634 if (e.getValue().equals(serverName)) {
635 it.remove();
636 }
637 }
638 long now = System.currentTimeMillis();
639 if (LOG.isDebugEnabled()) {
640 LOG.debug("Adding to log splitting servers " + serverName);
641 }
642 processedServers.put(serverName, Long.valueOf(now));
643 Configuration conf = server.getConfiguration();
644 long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME);
645
646 if (now > lastProcessedServerCleanTime + obsoleteTime) {
647 lastProcessedServerCleanTime = now;
648 long cutoff = now - obsoleteTime;
649 for (Iterator<Map.Entry<ServerName, Long>> it
650 = processedServers.entrySet().iterator(); it.hasNext();) {
651 Map.Entry<ServerName, Long> e = it.next();
652 if (e.getValue().longValue() < cutoff) {
653 if (LOG.isDebugEnabled()) {
654 LOG.debug("Removed from log splitting servers " + e.getKey());
655 }
656 it.remove();
657 }
658 }
659 }
660 }
661
662
663
664
665 public void logSplit(final HRegionInfo region) {
666 clearLastAssignment(region);
667 }
668
669 public synchronized void clearLastAssignment(final HRegionInfo region) {
670 lastAssignments.remove(region.getEncodedName());
671 }
672
673
674
675
676 public void regionOffline(final HRegionInfo hri) {
677 regionOffline(hri, null, false);
678 }
679
680
681
682
683
684
685 public void regionOffline(
686 final HRegionInfo hri, final State expectedState, final boolean force) {
687 Preconditions.checkArgument(expectedState == null
688 || RegionState.isUnassignable(expectedState),
689 "Offlined region should not be " + expectedState);
690 if (isRegionInState(hri, State.SPLITTING_NEW, State.MERGING_NEW)) {
691
692 deleteRegion(hri);
693 return;
694 }
695
696
697
698
699
700 State newState =
701 expectedState == null ? State.OFFLINE : expectedState;
702
703 if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) {
704 RegionState primateState = getRegionState(
705 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
706 if ((primateState != null) && (primateState.getState() == State.SPLIT)) {
707 if (LOG.isDebugEnabled()) {
708 LOG.debug("Update region " + hri + "to SPLIT, from primary region " +
709 RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
710 }
711 newState = State.SPLIT;
712 }
713 }
714
715 updateRegionState(hri, newState);
716 String encodedName = hri.getEncodedName();
717 synchronized (this) {
718 regionsInTransition.remove(encodedName);
719 ServerName oldServerName = regionAssignments.remove(hri);
720 if (oldServerName != null && serverHoldings.containsKey(oldServerName)) {
721 if (force || (newState == State.MERGED || newState == State.SPLIT
722 || hri.isMetaRegion() || tableStateManager.isTableState(hri.getTable(),
723 ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING))) {
724
725
726 LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
727 removeFromServerHoldings(oldServerName, hri);
728 removeFromReplicaMapping(hri);
729 } else {
730
731
732 oldAssignments.put(encodedName, oldServerName);
733 }
734 }
735 }
736 }
737
738
739
740
741 public List<HRegionInfo> serverOffline(final ZooKeeperWatcher watcher, final ServerName sn) {
742
743 List<HRegionInfo> rits = new ArrayList<HRegionInfo>();
744 Set<Pair<HRegionInfo, HRegionInfo>> regionsToClean =
745 new HashSet<Pair<HRegionInfo, HRegionInfo>>();
746
747
748
749 Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
750 Map<String, HRegionInfo> daughter2Parent = new HashMap<>();
751 synchronized (this) {
752 Set<HRegionInfo> assignedRegions = serverHoldings.get(sn);
753 if (assignedRegions == null) {
754 assignedRegions = new HashSet<HRegionInfo>();
755 }
756
757 for (HRegionInfo region : assignedRegions) {
758
759 if (isRegionOnline(region)) {
760 regionsToOffline.add(region);
761 } else if (isRegionInState(region, State.SPLITTING, State.MERGING)) {
762 LOG.debug("Offline splitting/merging region " + getRegionState(region));
763 try {
764
765 ZKAssign.deleteNodeFailSilent(watcher, region);
766 regionsToOffline.add(region);
767 PairOfSameType<HRegionInfo> daughterRegions =
768 MetaTableAccessor.getDaughterRegionsFromParent(this.server.getConnection(), region);
769 if (daughterRegions != null) {
770 if (daughterRegions.getFirst() != null) {
771 daughter2Parent.put(daughterRegions.getFirst().getEncodedName(), region);
772 }
773 if (daughterRegions.getSecond() != null) {
774 daughter2Parent.put(daughterRegions.getSecond().getEncodedName(), region);
775 }
776 }
777 } catch (KeeperException ke) {
778 server.abort("Unexpected ZK exception deleting node " + region, ke);
779 } catch (IOException e) {
780 LOG.warn("get daughter from meta exception " + region, e);
781 }
782 }
783 }
784
785 for (RegionState state : regionsInTransition.values()) {
786 HRegionInfo hri = state.getRegion();
787 if (assignedRegions.contains(hri)) {
788
789
790
791 LOG.info("Transitioning " + state + " will be handled by ServerCrashProcedure for " + sn);
792 } else if (sn.equals(state.getServerName())) {
793
794
795
796
797
798
799
800 if (state.isPendingOpenOrOpening() || state.isFailedClose() || state.isOffline()) {
801 LOG.info("Found region in " + state +
802 " to be reassigned by ServerCrashProcedure for " + sn);
803 rits.add(hri);
804 } else if (state.isSplittingNew() || state.isMergingNew()) {
805 LOG.info(
806 "Offline/Cleanup region if no meta entry exists, hri: " + hri + " state: " + state);
807 if (daughter2Parent.containsKey(hri.getEncodedName())) {
808 HRegionInfo parent = daughter2Parent.get(hri.getEncodedName());
809 HRegionInfo info = getHRIFromMeta(parent);
810 if (info != null && info.isSplit() && info.isOffline()) {
811 regionsToClean.add(Pair.newPair(state.getRegion(), info));
812 } else {
813 regionsToClean.add(Pair.<HRegionInfo, HRegionInfo>newPair(state.getRegion(), null));
814 }
815 } else {
816 regionsToClean.add(Pair.<HRegionInfo, HRegionInfo>newPair(state.getRegion(), null));
817 }
818 } else {
819 LOG.warn("THIS SHOULD NOT HAPPEN: unexpected " + state);
820 }
821 }
822 }
823 this.notifyAll();
824 }
825
826 for (HRegionInfo hri : regionsToOffline) {
827 regionOffline(hri);
828 }
829
830 cleanFailedSplitMergeRegions(regionsToClean);
831 return rits;
832 }
833
834 private HRegionInfo getHRIFromMeta(HRegionInfo parent) {
835 Result result = null;
836 try {
837 result =
838 MetaTableAccessor.getRegionResult(this.server.getConnection(), parent.getRegionName());
839 HRegionInfo info = MetaTableAccessor.getHRegionInfo(result);
840 return info;
841 } catch (IOException e) {
842 LOG.error("got exception when query meta with region " + parent.getEncodedName(), e);
843 return null;
844 }
845 }
846
847
848
849
850
851
852
853
854 private void cleanFailedSplitMergeRegions(Set<Pair<HRegionInfo, HRegionInfo>> hris) {
855 if (hris.isEmpty()) {
856 return;
857 }
858
859 for (Pair<HRegionInfo, HRegionInfo> hriPair : hris) {
860 HRegionInfo hri = hriPair.getFirst();
861 HRegionInfo parentInfo = hriPair.getSecond();
862
863
864
865 try {
866 Pair<HRegionInfo, ServerName> regionPair =
867 MetaTableAccessor.getRegion(server.getConnection(), hri.getRegionName());
868 if (regionPair == null || useZK) {
869 regionOffline(hri);
870
871
872 if (regionPair != null) {
873 MetaTableAccessor.deleteRegion(this.server.getConnection(), hri);
874 }
875 if (parentInfo != null) {
876 List<Mutation> mutations = new ArrayList<Mutation>();
877 HRegionInfo copyOfParent = new HRegionInfo(parentInfo);
878 copyOfParent.setOffline(false);
879 copyOfParent.setSplit(false);
880 Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
881 mutations.add(putParent);
882 MetaTableAccessor.mutateMetaTable(this.server.getConnection(), mutations);
883 }
884 LOG.debug("Cleaning up HDFS since no meta entry exists, hri: " + hri);
885 FSUtils.deleteRegionDir(server.getConfiguration(), hri);
886 }
887 } catch (IOException e) {
888 LOG.warn("Got exception while cleaning up region " + hri, e);
889 }
890 }
891 }
892
893
894
895
896
897
898
899
900
901
902
903 public synchronized List<HRegionInfo> getRegionsOfTable(TableName tableName) {
904 List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
905
906
907 HRegionInfo boundary = new HRegionInfo(tableName, null, null, false, 0L);
908 for (HRegionInfo hri: regionAssignments.tailMap(boundary).keySet()) {
909 if(!hri.getTable().equals(tableName)) break;
910 tableRegions.add(hri);
911 }
912 return tableRegions;
913 }
914
915
916
917
918
919
920
921
922
923
924 public synchronized Map<RegionState.State, List<HRegionInfo>>
925 getRegionByStateOfTable(TableName tableName) {
926 Map<RegionState.State, List<HRegionInfo>> tableRegions =
927 new HashMap<State, List<HRegionInfo>>();
928 for (State state : State.values()) {
929 tableRegions.put(state, new ArrayList<HRegionInfo>());
930 }
931 Map<String, RegionState> indexMap = regionStatesTableIndex.get(tableName);
932 if (indexMap == null)
933 return tableRegions;
934 for (RegionState regionState : indexMap.values()) {
935 tableRegions.get(regionState.getState()).add(regionState.getRegion());
936 }
937 return tableRegions;
938 }
939
940
941
942
943
944
945
946 public synchronized void waitOnRegionToClearRegionsInTransition(
947 final HRegionInfo hri) throws InterruptedException {
948 if (!isRegionInTransition(hri)) return;
949
950 while(!server.isStopped() && isRegionInTransition(hri)) {
951 RegionState rs = getRegionState(hri);
952 LOG.info("Waiting on " + rs + " to clear regions-in-transition");
953 waitForUpdate(100);
954 }
955
956 if (server.isStopped()) {
957 LOG.info("Giving up wait on region in " +
958 "transition because stoppable.isStopped is set");
959 }
960 }
961
962
963
964
965
966 public void tableDeleted(final TableName tableName) {
967 Set<HRegionInfo> regionsToDelete = new HashSet<HRegionInfo>();
968 synchronized (this) {
969 for (RegionState state: regionStates.values()) {
970 HRegionInfo region = state.getRegion();
971 if (region.getTable().equals(tableName)) {
972 regionsToDelete.add(region);
973 }
974 }
975 }
976 for (HRegionInfo region: regionsToDelete) {
977 deleteRegion(region);
978 }
979 }
980
981
982
983
984 public synchronized Set<HRegionInfo> getServerRegions(ServerName serverName) {
985 Set<HRegionInfo> regions = serverHoldings.get(serverName);
986 if (regions == null) return null;
987 return new HashSet<HRegionInfo>(regions);
988 }
989
990
991
992
993 public synchronized void deleteRegion(final HRegionInfo hri) {
994 String encodedName = hri.getEncodedName();
995 regionsInTransition.remove(encodedName);
996 regionStates.remove(encodedName);
997 TableName table = hri.getTable();
998 Map<String, RegionState> indexMap = regionStatesTableIndex.get(table);
999 indexMap.remove(encodedName);
1000 if (indexMap.size() == 0)
1001 regionStatesTableIndex.remove(table);
1002 lastAssignments.remove(encodedName);
1003 ServerName sn = regionAssignments.remove(hri);
1004 if (sn != null) {
1005 Set<HRegionInfo> regions = serverHoldings.get(sn);
1006 regions.remove(hri);
1007 }
1008 }
1009
1010 public boolean isRegionInRegionStates(final HRegionInfo hri) {
1011 return (getRegionState(hri) != null || isRegionOnline(hri)) || isRegionInTransition(hri)
1012 || isRegionInState(hri, State.OFFLINE, State.CLOSED);
1013 }
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025 synchronized boolean wasRegionOnDeadServer(final String encodedName) {
1026 ServerName server = lastAssignments.get(encodedName);
1027 return isServerDeadAndNotProcessed(server);
1028 }
1029
1030 synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
1031 if (server == null) return false;
1032 if (serverManager.isServerOnline(server)) {
1033 String hostAndPort = server.getHostAndPort();
1034 long startCode = server.getStartcode();
1035 Long deadCode = deadServers.get(hostAndPort);
1036 if (deadCode == null || startCode > deadCode.longValue()) {
1037 if (serverManager.isServerReachable(server)) {
1038 return false;
1039 }
1040
1041 deadServers.put(hostAndPort, Long.valueOf(startCode));
1042 }
1043
1044
1045
1046
1047
1048
1049
1050 LOG.warn("Couldn't reach online server " + server);
1051 }
1052
1053 return !processedServers.containsKey(server);
1054 }
1055
1056
1057
1058
1059
1060 synchronized ServerName getLastRegionServerOfRegion(final String encodedName) {
1061 return lastAssignments.get(encodedName);
1062 }
1063
1064 synchronized void setLastRegionServerOfRegions(
1065 final ServerName serverName, final List<HRegionInfo> regionInfos) {
1066 for (HRegionInfo hri: regionInfos) {
1067 setLastRegionServerOfRegion(serverName, hri.getEncodedName());
1068 }
1069 }
1070
1071 synchronized void setLastRegionServerOfRegion(
1072 final ServerName serverName, final String encodedName) {
1073 lastAssignments.put(encodedName, serverName);
1074 }
1075
1076 void splitRegion(HRegionInfo p,
1077 HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException {
1078
1079 regionStateStore.splitRegion(p, a, b, sn, getRegionReplication(p));
1080 synchronized (this) {
1081
1082
1083 Set<HRegionInfo> regions = serverHoldings.get(sn);
1084 if (regions == null) {
1085 throw new IllegalStateException(sn + " should host some regions");
1086 }
1087 regions.remove(p);
1088 regions.add(a);
1089 regions.add(b);
1090 }
1091 }
1092
1093 void mergeRegions(HRegionInfo p,
1094 HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException {
1095 regionStateStore.mergeRegions(p, a, b, sn, getRegionReplication(a));
1096 synchronized (this) {
1097
1098
1099 Set<HRegionInfo> regions = serverHoldings.get(sn);
1100 if (regions == null) {
1101 throw new IllegalStateException(sn + " should host some regions");
1102 }
1103 regions.remove(a);
1104 regions.remove(b);
1105 regions.add(p);
1106 }
1107 }
1108
1109 private int getRegionReplication(HRegionInfo r) throws IOException {
1110 if (tableStateManager != null) {
1111 HTableDescriptor htd = ((MasterServices)server).getTableDescriptors().get(r.getTable());
1112 if (htd != null) {
1113 return htd.getRegionReplication();
1114 }
1115 }
1116 return 1;
1117 }
1118
1119
1120
1121
1122
1123
1124 synchronized Map<HRegionInfo, ServerName> closeAllUserRegions(Set<TableName> excludedTables) {
1125 boolean noExcludeTables = excludedTables == null || excludedTables.isEmpty();
1126 Set<HRegionInfo> toBeClosed = new HashSet<HRegionInfo>(regionStates.size());
1127 for(RegionState state: regionStates.values()) {
1128 HRegionInfo hri = state.getRegion();
1129 if (state.isSplit() || hri.isSplit()) {
1130 continue;
1131 }
1132 TableName tableName = hri.getTable();
1133 if (!TableName.META_TABLE_NAME.equals(tableName)
1134 && (noExcludeTables || !excludedTables.contains(tableName))) {
1135 toBeClosed.add(hri);
1136 }
1137 }
1138 Map<HRegionInfo, ServerName> allUserRegions =
1139 new HashMap<HRegionInfo, ServerName>(toBeClosed.size());
1140 for (HRegionInfo hri: toBeClosed) {
1141 RegionState regionState = updateRegionState(hri, State.CLOSED);
1142 allUserRegions.put(hri, regionState.getServerName());
1143 }
1144 return allUserRegions;
1145 }
1146
1147
1148
1149
1150
1151
1152
1153 protected synchronized double getAverageLoad() {
1154 int numServers = 0, totalLoad = 0;
1155 for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1156 Set<HRegionInfo> regions = e.getValue();
1157 ServerName serverName = e.getKey();
1158 int regionCount = regions.size();
1159 if (serverManager.isServerOnline(serverName)) {
1160 totalLoad += regionCount;
1161 numServers++;
1162 }
1163 }
1164 if (numServers > 1) {
1165
1166
1167
1168 Set<HRegionInfo> hris = serverHoldings.get(server.getServerName());
1169 if (hris != null) {
1170 totalLoad -= hris.size();
1171 numServers--;
1172 }
1173 }
1174 return numServers == 0 ? 0.0 :
1175 (double)totalLoad / (double)numServers;
1176 }
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186 protected Map<TableName, Map<ServerName, List<HRegionInfo>>>
1187 getAssignmentsByTable() {
1188 Map<TableName, Map<ServerName, List<HRegionInfo>>> result =
1189 new HashMap<TableName, Map<ServerName,List<HRegionInfo>>>();
1190 synchronized (this) {
1191 if (!server.getConfiguration().getBoolean(
1192 HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false)) {
1193 Map<ServerName, List<HRegionInfo>> svrToRegions =
1194 new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1195 for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1196 svrToRegions.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
1197 }
1198 result.put(TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME), svrToRegions);
1199 } else {
1200 for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1201 for (HRegionInfo hri: e.getValue()) {
1202 if (hri.isMetaRegion()) continue;
1203 TableName tablename = hri.getTable();
1204 Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
1205 if (svrToRegions == null) {
1206 svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1207 result.put(tablename, svrToRegions);
1208 }
1209 List<HRegionInfo> regions = svrToRegions.get(e.getKey());
1210 if (regions == null) {
1211 regions = new ArrayList<HRegionInfo>();
1212 svrToRegions.put(e.getKey(), regions);
1213 }
1214 regions.add(hri);
1215 }
1216 }
1217 }
1218 }
1219
1220 Map<ServerName, ServerLoad>
1221 onlineSvrs = serverManager.getOnlineServers();
1222
1223 List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
1224 for (Map<ServerName, List<HRegionInfo>> map: result.values()) {
1225 for (ServerName svr: onlineSvrs.keySet()) {
1226 if (!map.containsKey(svr)) {
1227 map.put(svr, new ArrayList<HRegionInfo>());
1228 }
1229 }
1230 map.keySet().removeAll(drainingServers);
1231 }
1232 return result;
1233 }
1234
1235 protected RegionState getRegionState(final HRegionInfo hri) {
1236 return getRegionState(hri.getEncodedName());
1237 }
1238
1239
1240
1241
1242
1243 protected synchronized Map<ServerName, List<HRegionInfo>> getRegionAssignmentsByServer() {
1244 Map<ServerName, List<HRegionInfo>> regionsByServer =
1245 new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1246 for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1247 regionsByServer.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
1248 }
1249 return regionsByServer;
1250 }
1251
1252 protected synchronized RegionState getRegionState(final String encodedName) {
1253 return regionStates.get(encodedName);
1254 }
1255
1256
1257
1258
1259
1260
1261 @SuppressWarnings("deprecation")
1262 protected HRegionInfo getRegionInfo(final byte [] regionName) {
1263 String encodedName = HRegionInfo.encodeRegionName(regionName);
1264 RegionState regionState = getRegionState(encodedName);
1265 if (regionState != null) {
1266 return regionState.getRegion();
1267 }
1268
1269 try {
1270 Pair<HRegionInfo, ServerName> p =
1271 MetaTableAccessor.getRegion(server.getConnection(), regionName);
1272 HRegionInfo hri = p == null ? null : p.getFirst();
1273 if (hri != null) {
1274 createRegionState(hri);
1275 }
1276 return hri;
1277 } catch (IOException e) {
1278 server.abort("Aborting because error occurred while reading "
1279 + Bytes.toStringBinary(regionName) + " from hbase:meta", e);
1280 return null;
1281 }
1282 }
1283
1284 static boolean isOneOfStates(RegionState regionState, State... states) {
1285 State s = regionState != null ? regionState.getState() : null;
1286 for (State state: states) {
1287 if (s == state) return true;
1288 }
1289 return false;
1290 }
1291
1292
1293
1294
1295 private RegionState updateRegionState(final HRegionInfo hri,
1296 final State state, final ServerName serverName, long openSeqNum) {
1297 if (state == State.FAILED_CLOSE || state == State.FAILED_OPEN) {
1298 LOG.warn("Failed to open/close " + hri.getShortNameToLog()
1299 + " on " + serverName + ", set to " + state);
1300 }
1301
1302 String encodedName = hri.getEncodedName();
1303 RegionState regionState = new RegionState(
1304 hri, state, System.currentTimeMillis(), serverName);
1305 RegionState oldState = getRegionState(encodedName);
1306 if (!regionState.equals(oldState)) {
1307 LOG.info("Transition " + oldState + " to " + regionState);
1308
1309 regionStateStore.updateRegionState(openSeqNum, regionState, oldState);
1310 }
1311
1312 synchronized (this) {
1313 RegionState oldRegionState = regionsInTransition.put(encodedName, regionState);
1314
1315
1316 if (oldRegionState != null) {
1317 regionState.updateRitDuration(oldRegionState.getStamp());
1318 }
1319 putRegionState(regionState);
1320
1321
1322
1323 if ((state == State.CLOSED || state == State.MERGED
1324 || state == State.SPLIT) && lastAssignments.containsKey(encodedName)) {
1325 ServerName last = lastAssignments.get(encodedName);
1326 if (last.equals(serverName)) {
1327 lastAssignments.remove(encodedName);
1328 } else {
1329 LOG.warn(encodedName + " moved to " + state + " on "
1330 + serverName + ", expected " + last);
1331 }
1332 }
1333
1334
1335 if (serverName != null && state == State.OPEN) {
1336 ServerName last = lastAssignments.get(encodedName);
1337 if (!serverName.equals(last)) {
1338 lastAssignments.put(encodedName, serverName);
1339 if (last != null && isServerDeadAndNotProcessed(last)) {
1340 LOG.warn(encodedName + " moved to " + serverName
1341 + ", while it's previous host " + last
1342 + " is dead but not processed yet");
1343 }
1344 }
1345 }
1346
1347
1348 this.notifyAll();
1349 }
1350 return regionState;
1351 }
1352 }