View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import com.google.common.base.Preconditions;
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.HashSet;
26  import java.util.Iterator;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.Set;
30  import java.util.Comparator;
31  import java.util.SortedSet;
32  import java.util.TreeMap;
33  import java.util.TreeSet;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MetaTableAccessor;
42  import org.apache.hadoop.hbase.RegionTransition;
43  import org.apache.hadoop.hbase.ServerLoad;
44  import org.apache.hadoop.hbase.ServerName;
45  import org.apache.hadoop.hbase.TableName;
46  import org.apache.hadoop.hbase.TableStateManager;
47  import org.apache.hadoop.hbase.classification.InterfaceAudience;
48  import org.apache.hadoop.hbase.client.Mutation;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
51  import org.apache.hadoop.hbase.client.Result;
52  import org.apache.hadoop.hbase.master.RegionState.State;
53  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.ConfigUtil;
56  import org.apache.hadoop.hbase.util.FSUtils;
57  import org.apache.hadoop.hbase.util.Pair;
58  import org.apache.hadoop.hbase.util.PairOfSameType;
59  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
60  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
61  import org.apache.zookeeper.KeeperException;
62  
63  import com.google.common.annotations.VisibleForTesting;
64  import com.google.common.base.Preconditions;
65  
66  /**
67   * Region state accountant. It holds the states of all regions in the memory.
68   * In normal scenario, it should match the meta table and the true region states.
69   *
70   * This map is used by AssignmentManager to track region states.
71   */
72  @InterfaceAudience.Private
73  public class RegionStates {
74    private static final Log LOG = LogFactory.getLog(RegionStates.class);
75  
76    public final static RegionStateStampComparator REGION_STATE_COMPARATOR =
77      new RegionStateStampComparator();
78  
79    // This comparator sorts the RegionStates by time stamp then Region name.
80    // Comparing by timestamp alone can lead us to discard different RegionStates that happen
81    // to share a timestamp.
82    private static class RegionStateStampComparator implements Comparator<RegionState> {
83      @Override
84      public int compare(RegionState l, RegionState r) {
85        return Long.compare(l.getStamp(), r.getStamp()) == 0 ?
86            Bytes.compareTo(l.getRegion().getRegionName(), r.getRegion().getRegionName()) :
87            Long.compare(l.getStamp(), r.getStamp());
88      }
89    }
90  
91    /**
92     * Regions currently in transition.
93     */
94    final HashMap<String, RegionState> regionsInTransition =
95      new HashMap<String, RegionState>();
96  
97    /**
98     * Region encoded name to state map.
99     * All the regions should be in this map.
100    */
101   private final Map<String, RegionState> regionStates =
102     new HashMap<String, RegionState>();
103 
104   /**
105    * Holds mapping of table -> region state
106    */
107   private final Map<TableName, Map<String, RegionState>> regionStatesTableIndex =
108       new HashMap<TableName, Map<String, RegionState>>();
109 
110   /**
111    * Server to regions assignment map.
112    * Contains the set of regions currently assigned to a given server.
113    */
114   private final Map<ServerName, Set<HRegionInfo>> serverHoldings =
115     new HashMap<ServerName, Set<HRegionInfo>>();
116 
117   /**
118    * Maintains the mapping from the default region to the replica regions.
119    */
120   private final Map<HRegionInfo, Set<HRegionInfo>> defaultReplicaToOtherReplicas =
121     new HashMap<HRegionInfo, Set<HRegionInfo>>();
122 
123   /**
124    * Region to server assignment map.
125    * Contains the server a given region is currently assigned to.
126    */
127   private final TreeMap<HRegionInfo, ServerName> regionAssignments =
128     new TreeMap<HRegionInfo, ServerName>();
129 
130   /**
131    * Encoded region name to server assignment map for re-assignment
132    * purpose. Contains the server a given region is last known assigned
133    * to, which has not completed log splitting, so not assignable.
134    * If a region is currently assigned, this server info in this
135    * map should be the same as that in regionAssignments.
136    * However the info in regionAssignments is cleared when the region
137    * is offline while the info in lastAssignments is cleared when
138    * the region is closed or the server is dead and processed.
139    */
140   private final HashMap<String, ServerName> lastAssignments =
141     new HashMap<String, ServerName>();
142 
143   /**
144    * Encoded region name to server assignment map for the
145    * purpose to clean up serverHoldings when a region is online
146    * on a new server. When the region is offline from the previous
147    * server, we cleaned up regionAssignments so that it has the
148    * latest assignment map. But we didn't clean up serverHoldings
149    * to match the meta. We need this map to find out the old server
150    * whose serverHoldings needs cleanup, given a moved region.
151    */
152   private final HashMap<String, ServerName> oldAssignments =
153     new HashMap<String, ServerName>();
154 
155   /**
156    * Map a host port pair string to the latest start code
157    * of a region server which is known to be dead. It is dead
158    * to us, but server manager may not know it yet.
159    */
160   private final HashMap<String, Long> deadServers =
161     new HashMap<String, Long>();
162 
163   /**
164    * Map a dead servers to the time when log split is done.
165    * Since log splitting is not ordered, we have to remember
166    * all processed instances. The map is cleaned up based
167    * on a configured time. By default, we assume a dead
168    * server should be done with log splitting in two hours.
169    */
170   private final HashMap<ServerName, Long> processedServers =
171     new HashMap<ServerName, Long>();
172   private long lastProcessedServerCleanTime;
173 
174   private final TableStateManager tableStateManager;
175   private final RegionStateStore regionStateStore;
176   private final ServerManager serverManager;
177   private final MasterServices server;
178   private final boolean useZK; // Is it ZK based assignment?
179 
180   // The maximum time to keep a log split info in region states map
181   static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime";
182   static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours
183 
184   RegionStates(final MasterServices master, final TableStateManager tableStateManager,
185       final ServerManager serverManager, final RegionStateStore regionStateStore) {
186     this.tableStateManager = tableStateManager;
187     this.regionStateStore = regionStateStore;
188     this.serverManager = serverManager;
189     this.server = master;
190     this.useZK = ConfigUtil.useZKForAssignment(server.getConfiguration());
191   }
192 
193   /**
194    * @return a copy of the region assignment map
195    */
196   public synchronized Map<HRegionInfo, ServerName> getRegionAssignments() {
197     return new TreeMap<HRegionInfo, ServerName>(regionAssignments);
198   }
199 
200   /**
201    * Return the replicas (including default) for the regions grouped by ServerName
202    * @param regions
203    * @return a pair containing the groupings as a map
204    */
205   synchronized Map<ServerName, List<HRegionInfo>> getRegionAssignments(
206     Collection<HRegionInfo> regions) {
207     Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
208     for (HRegionInfo region : regions) {
209       HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(region);
210       Set<HRegionInfo> allReplicas = defaultReplicaToOtherReplicas.get(defaultReplica);
211       if (allReplicas != null) {
212         for (HRegionInfo hri : allReplicas) {
213           ServerName server = regionAssignments.get(hri);
214           if (server != null) {
215             List<HRegionInfo> regionsOnServer = map.get(server);
216             if (regionsOnServer == null) {
217               regionsOnServer = new ArrayList<HRegionInfo>(1);
218               map.put(server, regionsOnServer);
219             }
220             regionsOnServer.add(hri);
221           }
222         }
223       }
224     }
225     return map;
226   }
227 
228   public synchronized ServerName getRegionServerOfRegion(HRegionInfo hri) {
229     return regionAssignments.get(hri);
230   }
231 
232   /**
233    * Get regions in transition and their states
234    */
235   public synchronized Set<RegionState> getRegionsInTransition() {
236     return new HashSet<RegionState>(regionsInTransition.values());
237   }
238 
239   /**
240    * Get all regions and their states
241    */
242   public synchronized Set<RegionState> getAllRegions() {
243     return new HashSet<RegionState>(regionStates.values());
244   }
245 
246   /**
247    * @return a set of the regions in transition that are sorted by timestamp
248    */
249   public synchronized SortedSet<RegionState> getRegionsInTransitionOrderedByTimestamp() {
250     final TreeSet<RegionState> rit = new TreeSet<RegionState>(REGION_STATE_COMPARATOR);
251     for (RegionState rs: regionsInTransition.values()) {
252       rit.add(rs);
253     }
254     return rit;
255   }
256 
257   /**
258    * Get the number of regions in transition.
259    */
260   public synchronized int getRegionsInTransitionCount() {
261     return regionsInTransition.size();
262   }
263 
264   /**
265    * @return True if specified region in transition.
266    */
267   public synchronized boolean isRegionInTransition(final HRegionInfo hri) {
268     return regionsInTransition.containsKey(hri.getEncodedName());
269   }
270 
271   /**
272    * @return True if specified region in transition.
273    */
274   public synchronized boolean isRegionInTransition(final String encodedName) {
275     return regionsInTransition.containsKey(encodedName);
276   }
277 
278   /**
279    * @return True if any region in transition.
280    */
281   public synchronized boolean isRegionsInTransition() {
282     return !regionsInTransition.isEmpty();
283   }
284 
285   /**
286    * @return True if hbase:meta table region is in transition.
287    */
288   public synchronized boolean isMetaRegionInTransition() {
289     for (RegionState state : regionsInTransition.values()) {
290       if (state.getRegion().isMetaRegion()) return true;
291     }
292     return false;
293   }
294 
295   /**
296    * @return True if specified region assigned, and not in transition.
297    */
298   public synchronized boolean isRegionOnline(final HRegionInfo hri) {
299     return !isRegionInTransition(hri) && regionAssignments.containsKey(hri);
300   }
301 
302   /**
303    * @return True if specified region offline/closed, but not in transition.
304    * If the region is not in the map, it is offline to us too.
305    */
306   public synchronized boolean isRegionOffline(final HRegionInfo hri) {
307     return getRegionState(hri) == null || (!isRegionInTransition(hri)
308       && isRegionInState(hri, State.OFFLINE, State.CLOSED));
309   }
310 
311   /**
312    * @return True if specified region is in one of the specified states.
313    */
314   public boolean isRegionInState(
315       final HRegionInfo hri, final State... states) {
316     return isRegionInState(hri.getEncodedName(), states);
317   }
318 
319   /**
320    * @return True if specified region is in one of the specified states.
321    */
322   public boolean isRegionInState(
323       final String encodedName, final State... states) {
324     RegionState regionState = getRegionState(encodedName);
325     return isOneOfStates(regionState, states);
326   }
327 
328   /**
329    * Wait for the state map to be updated by assignment manager.
330    */
331   public synchronized void waitForUpdate(
332       final long timeout) throws InterruptedException {
333     this.wait(timeout);
334   }
335 
336   /**
337    * Get region transition state
338    */
339   public RegionState getRegionTransitionState(final HRegionInfo hri) {
340     return getRegionTransitionState(hri.getEncodedName());
341   }
342 
343   /**
344    * Get region transition state
345    */
346   public synchronized RegionState
347       getRegionTransitionState(final String encodedName) {
348     return regionsInTransition.get(encodedName);
349   }
350 
351   /**
352    * Add a list of regions to RegionStates. If a region is split
353    * and offline, its state will be SPLIT. Otherwise, its state will
354    * be OFFLINE. Region already in RegionStates will be skipped.
355    */
356   public void createRegionStates(
357       final List<HRegionInfo> hris) {
358     for (HRegionInfo hri: hris) {
359       createRegionState(hri);
360     }
361   }
362 
363   /**
364    * Add a region to RegionStates. If the region is split
365    * and offline, its state will be SPLIT. Otherwise, its state will
366    * be OFFLINE. If it is already in RegionStates, this call has
367    * no effect, and the original state is returned.
368    */
369   public RegionState createRegionState(final HRegionInfo hri) {
370     return createRegionState(hri, null, null, null);
371   }
372 
373   /**
374    * Add a region to RegionStates with the specified state.
375    * If the region is already in RegionStates, this call has
376    * no effect, and the original state is returned.
377    *
378    * @param hri the region info to create a state for
379    * @param newState the state to the region in set to
380    * @param serverName the server the region is transitioning on
381    * @param lastHost the last server that hosts the region
382    * @return the current state
383    */
384   public synchronized RegionState createRegionState(final HRegionInfo hri,
385       State newState, ServerName serverName, ServerName lastHost) {
386     if (newState == null || (newState == State.OPEN && serverName == null)) {
387       newState =  State.OFFLINE;
388     }
389     if (hri.isOffline() && hri.isSplit()) {
390       newState = State.SPLIT;
391       serverName = null;
392     }
393     String encodedName = hri.getEncodedName();
394     RegionState regionState = regionStates.get(encodedName);
395     if (regionState != null) {
396       LOG.warn("Tried to create a state for a region already in RegionStates, "
397         + "used existing: " + regionState + ", ignored new: " + newState);
398     } else {
399       regionState = new RegionState(hri, newState, serverName);
400       putRegionState(regionState);
401       if (newState == State.OPEN) {
402         if (!serverName.equals(lastHost)) {
403           LOG.warn("Open region's last host " + lastHost
404             + " should be the same as the current one " + serverName
405             + ", ignored the last and used the current one");
406           lastHost = serverName;
407         }
408         lastAssignments.put(encodedName, lastHost);
409         regionAssignments.put(hri, lastHost);
410       } else if (!regionState.isUnassignable()) {
411         regionsInTransition.put(encodedName, regionState);
412       }
413       if (lastHost != null && newState != State.SPLIT) {
414         addToReplicaMapping(hri);
415         addToServerHoldings(lastHost, hri);
416         if (newState != State.OPEN) {
417           oldAssignments.put(encodedName, lastHost);
418         }
419       }
420     }
421     return regionState;
422   }
423 
424   private RegionState putRegionState(RegionState regionState) {
425     HRegionInfo hri = regionState.getRegion();
426     String encodedName = hri.getEncodedName();
427     TableName table = hri.getTable();
428     RegionState oldState = regionStates.put(encodedName, regionState);
429     Map<String, RegionState> map = regionStatesTableIndex.get(table);
430     if (map == null) {
431       map = new HashMap<String, RegionState>();
432       regionStatesTableIndex.put(table, map);
433     }
434     map.put(encodedName, regionState);
435     return oldState;
436   }
437 
438   /**
439    * Set the region state to CLOSED
440    */
441   public RegionState setRegionStateTOCLOSED(
442       final byte[] regionName,
443       final ServerName serverName) {
444     HRegionInfo regionInfo = getRegionInfo(regionName);
445     return setRegionStateTOCLOSED(regionInfo, serverName);
446   }
447 
448   /**
449    * Set the region state to CLOSED
450    */
451   public RegionState setRegionStateTOCLOSED(
452       final HRegionInfo regionInfo,
453       final ServerName serverName) {
454     ServerName sn = serverName;
455     if (sn == null) {
456       RegionState regionState = getRegionState(regionInfo.getEncodedName());
457       if (regionState != null) {
458         sn = regionState.getServerName();
459       }
460       // TODO: if sn is null, should we dig into
461       // lastAssignments.get(regionInfo.getEncodedName() to get the server name?
462       // For now, I just keep the same logic that works in the past
463     }
464     // We have to make sure that the last region server is set to be the same as the
465     // current RS.  If we don't do that, we could run into situation that both AM and SSH
466     // think other would do the assignment work; at the end, neither does the work and
467     // region remains RIT.
468     // See HBASE-13330 and HBASE-17023
469     setLastRegionServerOfRegion(sn, regionInfo.getEncodedName());
470     return updateRegionState(regionInfo, State.CLOSED, sn);
471   }
472 
473   /**
474    * Update a region state. It will be put in transition if not already there.
475    */
476   public RegionState updateRegionState(
477       final HRegionInfo hri, final State state) {
478     RegionState regionState = getRegionState(hri.getEncodedName());
479     return updateRegionState(hri, state,
480       regionState == null ? null : regionState.getServerName());
481   }
482 
483   /**
484    * Update a region state. It will be put in transition if not already there.
485    *
486    * If we can't find the region info based on the region name in
487    * the transition, log a warning and return null.
488    */
489   public RegionState updateRegionState(
490       final RegionTransition transition, final State state) {
491     byte [] regionName = transition.getRegionName();
492     HRegionInfo regionInfo = getRegionInfo(regionName);
493     if (regionInfo == null) {
494       String prettyRegionName = HRegionInfo.prettyPrint(
495         HRegionInfo.encodeRegionName(regionName));
496       LOG.warn("Failed to find region " + prettyRegionName
497         + " in updating its state to " + state
498         + " based on region transition " + transition);
499       return null;
500     }
501     return updateRegionState(regionInfo, state,
502       transition.getServerName());
503   }
504 
505   /**
506    * Transition a region state to OPEN from OPENING/PENDING_OPEN
507    */
508   public synchronized RegionState transitionOpenFromPendingOpenOrOpeningOnServer(
509       final RegionTransition transition, final RegionState fromState, final ServerName sn) {
510     if(fromState.isPendingOpenOrOpeningOnServer(sn)){
511       return updateRegionState(transition, State.OPEN);
512     }
513     return null;
514   }
515 
516   /**
517    * Update a region state. It will be put in transition if not already there.
518    */
519   public RegionState updateRegionState(
520       final HRegionInfo hri, final State state, final ServerName serverName) {
521     return updateRegionState(hri, state, serverName, HConstants.NO_SEQNUM);
522   }
523 
524   public void regionOnline(final HRegionInfo hri, final ServerName serverName) {
525     regionOnline(hri, serverName, HConstants.NO_SEQNUM);
526   }
527 
528   /**
529    * A region is online, won't be in transition any more.
530    * We can't confirm it is really online on specified region server
531    * because it hasn't been put in region server's online region list yet.
532    */
533   public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) {
534     String encodedName = hri.getEncodedName();
535     if (!serverManager.isServerOnline(serverName)) {
536       // This is possible if the region server dies before master gets a
537       // chance to handle ZK event in time. At this time, if the dead server
538       // is already processed by SSH, we should ignore this event.
539       // If not processed yet, ignore and let SSH deal with it.
540       LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName);
541       return;
542     }
543     updateRegionState(hri, State.OPEN, serverName, openSeqNum);
544 
545     synchronized (this) {
546       RegionState regionState = regionsInTransition.remove(encodedName);
547       // When region is online and remove from regionsInTransition,
548       // update the RIT duration to assignment manager metrics
549       if (regionState != null && this.server.getAssignmentManager() != null) {
550         long ritDuration = System.currentTimeMillis() - regionState.getStamp()
551             + regionState.getRitDuration();
552         this.server.getAssignmentManager().getAssignmentManagerMetrics()
553             .updateRitDuration(ritDuration);
554       }
555       ServerName oldServerName = regionAssignments.put(hri, serverName);
556       if (!serverName.equals(oldServerName)) {
557         if (LOG.isDebugEnabled()) {
558           LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName);
559         }
560         addToServerHoldings(serverName, hri);
561         addToReplicaMapping(hri);
562         if (oldServerName == null) {
563           oldServerName = oldAssignments.remove(encodedName);
564         }
565         if (oldServerName != null
566             && !oldServerName.equals(serverName)
567             && serverHoldings.containsKey(oldServerName)) {
568           LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
569           removeFromServerHoldings(oldServerName, hri);
570         }
571       }
572     }
573   }
574 
575   private void addToServerHoldings(ServerName serverName, HRegionInfo hri) {
576     Set<HRegionInfo> regions = serverHoldings.get(serverName);
577     if (regions == null) {
578       regions = new HashSet<HRegionInfo>();
579       serverHoldings.put(serverName, regions);
580     }
581     regions.add(hri);
582   }
583 
584   private void addToReplicaMapping(HRegionInfo hri) {
585     HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
586     Set<HRegionInfo> replicas =
587         defaultReplicaToOtherReplicas.get(defaultReplica);
588     if (replicas == null) {
589       replicas = new HashSet<HRegionInfo>();
590       defaultReplicaToOtherReplicas.put(defaultReplica, replicas);
591     }
592     replicas.add(hri);
593   }
594 
595   private void removeFromServerHoldings(ServerName serverName, HRegionInfo hri) {
596     Set<HRegionInfo> oldRegions = serverHoldings.get(serverName);
597     oldRegions.remove(hri);
598     if (oldRegions.isEmpty()) {
599       serverHoldings.remove(serverName);
600     }
601   }
602 
603   private void removeFromReplicaMapping(HRegionInfo hri) {
604     HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
605     Set<HRegionInfo> replicas = defaultReplicaToOtherReplicas.get(defaultReplica);
606     if (replicas != null) {
607       replicas.remove(hri);
608       if (replicas.isEmpty()) {
609         defaultReplicaToOtherReplicas.remove(defaultReplica);
610       }
611     }
612   }
613 
614   /**
615    * Used in some unit tests
616    */
617   synchronized boolean existsInServerHoldings(final ServerName serverName,
618       final HRegionInfo hri) {
619     Set<HRegionInfo> oldRegions = serverHoldings.get(serverName);
620     if (oldRegions != null) {
621       return oldRegions.contains(hri);
622     }
623     return false;
624   }
625 
626   /**
627    * A dead server's wals have been split so that all the regions
628    * used to be open on it can be safely assigned now. Mark them assignable.
629    */
630   public synchronized void logSplit(final ServerName serverName) {
631     for (Iterator<Map.Entry<String, ServerName>> it
632         = lastAssignments.entrySet().iterator(); it.hasNext();) {
633       Map.Entry<String, ServerName> e = it.next();
634       if (e.getValue().equals(serverName)) {
635         it.remove();
636       }
637     }
638     long now = System.currentTimeMillis();
639     if (LOG.isDebugEnabled()) {
640       LOG.debug("Adding to log splitting servers " + serverName);
641     }
642     processedServers.put(serverName, Long.valueOf(now));
643     Configuration conf = server.getConfiguration();
644     long obsoleteTime = conf.getLong(LOG_SPLIT_TIME, DEFAULT_LOG_SPLIT_TIME);
645     // Doesn't have to be very accurate about the clean up time
646     if (now > lastProcessedServerCleanTime + obsoleteTime) {
647       lastProcessedServerCleanTime = now;
648       long cutoff = now - obsoleteTime;
649       for (Iterator<Map.Entry<ServerName, Long>> it
650           = processedServers.entrySet().iterator(); it.hasNext();) {
651         Map.Entry<ServerName, Long> e = it.next();
652         if (e.getValue().longValue() < cutoff) {
653           if (LOG.isDebugEnabled()) {
654             LOG.debug("Removed from log splitting servers " + e.getKey());
655           }
656           it.remove();
657         }
658       }
659     }
660   }
661 
662   /**
663    * Log split is done for a given region, so it is assignable now.
664    */
665   public void logSplit(final HRegionInfo region) {
666     clearLastAssignment(region);
667   }
668 
669   public synchronized void clearLastAssignment(final HRegionInfo region) {
670     lastAssignments.remove(region.getEncodedName());
671   }
672 
673   /**
674    * A region is offline, won't be in transition any more.
675    */
676   public void regionOffline(final HRegionInfo hri) {
677     regionOffline(hri, null, false);
678   }
679 
680   /**
681    * A region is offline, won't be in transition any more. Its state
682    * should be the specified expected state, which can only be
683    * Split/Merged/Offline/null(=Offline)/SplittingNew/MergingNew.
684    */
685   public void regionOffline(
686       final HRegionInfo hri, final State expectedState, final boolean force) {
687     Preconditions.checkArgument(expectedState == null
688       || RegionState.isUnassignable(expectedState),
689         "Offlined region should not be " + expectedState);
690     if (isRegionInState(hri, State.SPLITTING_NEW, State.MERGING_NEW)) {
691       // Remove it from all region maps
692       deleteRegion(hri);
693       return;
694     }
695 
696     /*
697      * One tricky case, if region here is a replica region and its parent is at
698      * SPLIT state, its newState should be same as its parent, not OFFLINE.
699      */
700     State newState =
701         expectedState == null ? State.OFFLINE : expectedState;
702 
703     if ((expectedState == null) && !RegionReplicaUtil.isDefaultReplica(hri)) {
704       RegionState primateState = getRegionState(
705           RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
706       if ((primateState != null) && (primateState.getState() == State.SPLIT)) {
707         if (LOG.isDebugEnabled()) {
708           LOG.debug("Update region " + hri + "to SPLIT, from primary region " +
709               RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
710         }
711         newState = State.SPLIT;
712       }
713     }
714 
715     updateRegionState(hri, newState);
716     String encodedName = hri.getEncodedName();
717     synchronized (this) {
718       regionsInTransition.remove(encodedName);
719       ServerName oldServerName = regionAssignments.remove(hri);
720       if (oldServerName != null && serverHoldings.containsKey(oldServerName)) {
721         if (force || (newState == State.MERGED || newState == State.SPLIT
722             || hri.isMetaRegion() || tableStateManager.isTableState(hri.getTable(),
723               ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING))) {
724           // Offline the region only if it's merged/split, or the table is disabled/disabling.
725           // Otherwise, offline it from this server only when it is online on a different server.
726           LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
727           removeFromServerHoldings(oldServerName, hri);
728           removeFromReplicaMapping(hri);
729         } else {
730           // Need to remember it so that we can offline it from this
731           // server when it is online on a different server.
732           oldAssignments.put(encodedName, oldServerName);
733         }
734       }
735     }
736   }
737 
738   /**
739    * A server is offline, all regions on it are dead.
740    */
741   public List<HRegionInfo> serverOffline(final ZooKeeperWatcher watcher, final ServerName sn) {
742     // Offline all regions on this server not already in transition.
743     List<HRegionInfo> rits = new ArrayList<HRegionInfo>();
744     Set<Pair<HRegionInfo, HRegionInfo>> regionsToClean =
745       new HashSet<Pair<HRegionInfo, HRegionInfo>>();
746     // Offline regions outside the loop and synchronized block to avoid
747     // ConcurrentModificationException and deadlock in case of meta anassigned,
748     // but RegionState a blocked.
749     Set<HRegionInfo> regionsToOffline = new HashSet<HRegionInfo>();
750     Map<String, HRegionInfo> daughter2Parent = new HashMap<>();
751     synchronized (this) {
752       Set<HRegionInfo> assignedRegions = serverHoldings.get(sn);
753       if (assignedRegions == null) {
754         assignedRegions = new HashSet<HRegionInfo>();
755       }
756 
757       for (HRegionInfo region : assignedRegions) {
758         // Offline open regions, no need to offline if SPLIT/MERGED/OFFLINE
759         if (isRegionOnline(region)) {
760           regionsToOffline.add(region);
761         } else if (isRegionInState(region, State.SPLITTING, State.MERGING)) {
762           LOG.debug("Offline splitting/merging region " + getRegionState(region));
763           try {
764             // Delete the ZNode if exists
765             ZKAssign.deleteNodeFailSilent(watcher, region);
766             regionsToOffline.add(region);
767             PairOfSameType<HRegionInfo> daughterRegions =
768               MetaTableAccessor.getDaughterRegionsFromParent(this.server.getConnection(), region);
769             if (daughterRegions != null) {
770               if (daughterRegions.getFirst() != null) {
771                 daughter2Parent.put(daughterRegions.getFirst().getEncodedName(), region);
772               }
773               if (daughterRegions.getSecond() != null) {
774                 daughter2Parent.put(daughterRegions.getSecond().getEncodedName(), region);
775               }
776             }
777           } catch (KeeperException ke) {
778             server.abort("Unexpected ZK exception deleting node " + region, ke);
779           } catch (IOException e) {
780             LOG.warn("get daughter from meta exception " + region, e);
781           }
782         }
783       }
784 
785       for (RegionState state : regionsInTransition.values()) {
786         HRegionInfo hri = state.getRegion();
787         if (assignedRegions.contains(hri)) {
788           // Region is open on this region server, but in transition.
789           // This region must be moving away from this server, or splitting/merging.
790           // SSH will handle it, either skip assigning, or re-assign.
791           LOG.info("Transitioning " + state + " will be handled by ServerCrashProcedure for " + sn);
792         } else if (sn.equals(state.getServerName())) {
793           // Region is in transition on this region server, and this
794           // region is not open on this server. So the region must be
795           // moving to this server from another one (i.e. opening or
796           // pending open on this server, was open on another one.
797           // Offline state is also kind of pending open if the region is in
798           // transition. The region could be in failed_close state too if we have
799           // tried several times to open it while this region server is not reachable)
800           if (state.isPendingOpenOrOpening() || state.isFailedClose() || state.isOffline()) {
801             LOG.info("Found region in " + state +
802               " to be reassigned by ServerCrashProcedure for " + sn);
803             rits.add(hri);
804           } else if (state.isSplittingNew() || state.isMergingNew()) {
805             LOG.info(
806               "Offline/Cleanup region if no meta entry exists, hri: " + hri + " state: " + state);
807             if (daughter2Parent.containsKey(hri.getEncodedName())) {
808               HRegionInfo parent = daughter2Parent.get(hri.getEncodedName());
809               HRegionInfo info = getHRIFromMeta(parent);
810               if (info != null && info.isSplit() && info.isOffline()) {
811                 regionsToClean.add(Pair.newPair(state.getRegion(), info));
812               } else {
813                 regionsToClean.add(Pair.<HRegionInfo, HRegionInfo>newPair(state.getRegion(), null));
814               }
815             } else {
816               regionsToClean.add(Pair.<HRegionInfo, HRegionInfo>newPair(state.getRegion(), null));
817             }
818           } else {
819             LOG.warn("THIS SHOULD NOT HAPPEN: unexpected " + state);
820           }
821         }
822       }
823       this.notifyAll();
824     }
825 
826     for (HRegionInfo hri : regionsToOffline) {
827       regionOffline(hri);
828     }
829 
830     cleanFailedSplitMergeRegions(regionsToClean);
831     return rits;
832   }
833 
834   private HRegionInfo getHRIFromMeta(HRegionInfo parent) {
835     Result result = null;
836     try {
837       result =
838         MetaTableAccessor.getRegionResult(this.server.getConnection(), parent.getRegionName());
839       HRegionInfo info = MetaTableAccessor.getHRegionInfo(result);
840       return info;
841     } catch (IOException e) {
842       LOG.error("got exception when query meta with region " + parent.getEncodedName(), e);
843       return null;
844     }
845   }
846 
847   /**
848    * This method does an RPC to hbase:meta. Do not call this method with a lock/synchronize held.
849    * In ZK mode we rollback and hence cleanup daughters/merged region. We also cleanup if
850    * meta doesn't have these regions.
851    *
852    * @param hris The hris to check if empty in hbase:meta and if so, clean them up.
853    */
854   private void cleanFailedSplitMergeRegions(Set<Pair<HRegionInfo, HRegionInfo>> hris) {
855     if (hris.isEmpty()) {
856       return;
857     }
858 
859     for (Pair<HRegionInfo, HRegionInfo> hriPair : hris) {
860       HRegionInfo hri = hriPair.getFirst();
861       HRegionInfo parentInfo = hriPair.getSecond();
862       // This is RPC to meta table. It is done while we have a synchronize on
863       // regionstates. No progress will be made if meta is not available at this time.
864       // This is a cleanup task. Not critical.
865       try {
866         Pair<HRegionInfo, ServerName> regionPair =
867             MetaTableAccessor.getRegion(server.getConnection(), hri.getRegionName());
868         if (regionPair == null || useZK) {
869           regionOffline(hri);
870 
871           // If we use ZK, then we can cleanup entries from meta, since we roll back.
872           if (regionPair != null) {
873             MetaTableAccessor.deleteRegion(this.server.getConnection(), hri);
874           }
875           if (parentInfo != null) {
876             List<Mutation> mutations = new ArrayList<Mutation>();
877             HRegionInfo copyOfParent = new HRegionInfo(parentInfo);
878             copyOfParent.setOffline(false);
879             copyOfParent.setSplit(false);
880             Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
881             mutations.add(putParent);
882             MetaTableAccessor.mutateMetaTable(this.server.getConnection(), mutations);
883           }
884           LOG.debug("Cleaning up HDFS since no meta entry exists, hri: " + hri);
885           FSUtils.deleteRegionDir(server.getConfiguration(), hri);
886         }
887       } catch (IOException e) {
888         LOG.warn("Got exception while cleaning up region " + hri, e);
889       }
890     }
891   }
892 
893   /**
894    * Gets the online regions of the specified table.
895    * This method looks at the in-memory state.  It does not go to <code>hbase:meta</code>.
896    * Only returns <em>online</em> regions.  If a region on this table has been
897    * closed during a disable, etc., it will be included in the returned list.
898    * So, the returned list may not necessarily be ALL regions in this table, its
899    * all the ONLINE regions in the table.
900    * @param tableName
901    * @return Online regions from <code>tableName</code>
902    */
903   public synchronized List<HRegionInfo> getRegionsOfTable(TableName tableName) {
904     List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
905     // boundary needs to have table's name but regionID 0 so that it is sorted
906     // before all table's regions.
907     HRegionInfo boundary = new HRegionInfo(tableName, null, null, false, 0L);
908     for (HRegionInfo hri: regionAssignments.tailMap(boundary).keySet()) {
909       if(!hri.getTable().equals(tableName)) break;
910       tableRegions.add(hri);
911     }
912     return tableRegions;
913   }
914 
915   /**
916    * Gets current state of all regions of the table.
917    * This method looks at the in-memory state.  It does not go to <code>hbase:meta</code>.
918    * Method guaranteed to return keys for all states
919    * in {@link org.apache.hadoop.hbase.master.RegionState.State}
920    *
921    * @param tableName
922    * @return Online regions from <code>tableName</code>
923    */
924   public synchronized Map<RegionState.State, List<HRegionInfo>>
925   getRegionByStateOfTable(TableName tableName) {
926     Map<RegionState.State, List<HRegionInfo>> tableRegions =
927         new HashMap<State, List<HRegionInfo>>();
928     for (State state : State.values()) {
929       tableRegions.put(state, new ArrayList<HRegionInfo>());
930     }
931     Map<String, RegionState> indexMap = regionStatesTableIndex.get(tableName);
932     if (indexMap == null)
933       return tableRegions;
934     for (RegionState regionState : indexMap.values()) {
935       tableRegions.get(regionState.getState()).add(regionState.getRegion());
936     }
937     return tableRegions;
938   }
939 
940   /**
941    * Wait on region to clear regions-in-transition.
942    * <p>
943    * If the region isn't in transition, returns immediately.  Otherwise, method
944    * blocks until the region is out of transition.
945    */
946   public synchronized void waitOnRegionToClearRegionsInTransition(
947       final HRegionInfo hri) throws InterruptedException {
948     if (!isRegionInTransition(hri)) return;
949 
950     while(!server.isStopped() && isRegionInTransition(hri)) {
951       RegionState rs = getRegionState(hri);
952       LOG.info("Waiting on " + rs + " to clear regions-in-transition");
953       waitForUpdate(100);
954     }
955 
956     if (server.isStopped()) {
957       LOG.info("Giving up wait on region in " +
958         "transition because stoppable.isStopped is set");
959     }
960   }
961 
962   /**
963    * A table is deleted. Remove its regions from all internal maps.
964    * We loop through all regions assuming we don't delete tables too much.
965    */
966   public void tableDeleted(final TableName tableName) {
967     Set<HRegionInfo> regionsToDelete = new HashSet<HRegionInfo>();
968     synchronized (this) {
969       for (RegionState state: regionStates.values()) {
970         HRegionInfo region = state.getRegion();
971         if (region.getTable().equals(tableName)) {
972           regionsToDelete.add(region);
973         }
974       }
975     }
976     for (HRegionInfo region: regionsToDelete) {
977       deleteRegion(region);
978     }
979   }
980 
981   /**
982    * Get a copy of all regions assigned to a server
983    */
984   public synchronized Set<HRegionInfo> getServerRegions(ServerName serverName) {
985     Set<HRegionInfo> regions = serverHoldings.get(serverName);
986     if (regions == null) return null;
987     return new HashSet<HRegionInfo>(regions);
988   }
989 
990   /**
991    * Remove a region from all state maps.
992    */
993   public synchronized void deleteRegion(final HRegionInfo hri) {
994     String encodedName = hri.getEncodedName();
995     regionsInTransition.remove(encodedName);
996     regionStates.remove(encodedName);
997     TableName table = hri.getTable();
998     Map<String, RegionState> indexMap = regionStatesTableIndex.get(table);
999     indexMap.remove(encodedName);
1000     if (indexMap.size() == 0)
1001       regionStatesTableIndex.remove(table);
1002     lastAssignments.remove(encodedName);
1003     ServerName sn = regionAssignments.remove(hri);
1004     if (sn != null) {
1005       Set<HRegionInfo> regions = serverHoldings.get(sn);
1006       regions.remove(hri);
1007     }
1008   }
1009 
1010   public boolean isRegionInRegionStates(final HRegionInfo hri) {
1011     return (getRegionState(hri) != null || isRegionOnline(hri)) || isRegionInTransition(hri)
1012         || isRegionInState(hri, State.OFFLINE, State.CLOSED);
1013      }
1014 
1015   /**
1016    * Checking if a region was assigned to a server which is not online now.
1017    * If so, we should hold re-assign this region till SSH has split its wals.
1018    * Once logs are split, the last assignment of this region will be reset,
1019    * which means a null last assignment server is ok for re-assigning.
1020    *
1021    * A region server could be dead but we don't know it yet. We may
1022    * think it's online falsely. Therefore if a server is online, we still
1023    * need to confirm it reachable and having the expected start code.
1024    */
1025   synchronized boolean wasRegionOnDeadServer(final String encodedName) {
1026     ServerName server = lastAssignments.get(encodedName);
1027     return isServerDeadAndNotProcessed(server);
1028   }
1029 
1030   synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
1031     if (server == null) return false;
1032     if (serverManager.isServerOnline(server)) {
1033       String hostAndPort = server.getHostAndPort();
1034       long startCode = server.getStartcode();
1035       Long deadCode = deadServers.get(hostAndPort);
1036       if (deadCode == null || startCode > deadCode.longValue()) {
1037         if (serverManager.isServerReachable(server)) {
1038           return false;
1039         }
1040         // The size of deadServers won't grow unbounded.
1041         deadServers.put(hostAndPort, Long.valueOf(startCode));
1042       }
1043       // Watch out! If the server is not dead, the region could
1044       // remain unassigned. That's why ServerManager#isServerReachable
1045       // should use some retry.
1046       //
1047       // We cache this info since it is very unlikely for that
1048       // instance to come back up later on. We don't want to expire
1049       // the server since we prefer to let it die naturally.
1050       LOG.warn("Couldn't reach online server " + server);
1051     }
1052     // Now, we know it's dead. Check if it's processed
1053     return !processedServers.containsKey(server);
1054   }
1055 
1056  /**
1057    * Get the last region server a region was on for purpose of re-assignment,
1058    * i.e. should the re-assignment be held back till log split is done?
1059    */
1060   synchronized ServerName getLastRegionServerOfRegion(final String encodedName) {
1061     return lastAssignments.get(encodedName);
1062   }
1063 
1064   synchronized void setLastRegionServerOfRegions(
1065       final ServerName serverName, final List<HRegionInfo> regionInfos) {
1066     for (HRegionInfo hri: regionInfos) {
1067       setLastRegionServerOfRegion(serverName, hri.getEncodedName());
1068     }
1069   }
1070 
1071   synchronized void setLastRegionServerOfRegion(
1072       final ServerName serverName, final String encodedName) {
1073     lastAssignments.put(encodedName, serverName);
1074   }
1075 
1076   void splitRegion(HRegionInfo p,
1077       HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException {
1078 
1079     regionStateStore.splitRegion(p, a, b, sn, getRegionReplication(p));
1080     synchronized (this) {
1081       // After PONR, split is considered to be done.
1082       // Update server holdings to be aligned with the meta.
1083       Set<HRegionInfo> regions = serverHoldings.get(sn);
1084       if (regions == null) {
1085         throw new IllegalStateException(sn + " should host some regions");
1086       }
1087       regions.remove(p);
1088       regions.add(a);
1089       regions.add(b);
1090     }
1091   }
1092 
1093   void mergeRegions(HRegionInfo p,
1094       HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException {
1095     regionStateStore.mergeRegions(p, a, b, sn, getRegionReplication(a));
1096     synchronized (this) {
1097       // After PONR, merge is considered to be done.
1098       // Update server holdings to be aligned with the meta.
1099       Set<HRegionInfo> regions = serverHoldings.get(sn);
1100       if (regions == null) {
1101         throw new IllegalStateException(sn + " should host some regions");
1102       }
1103       regions.remove(a);
1104       regions.remove(b);
1105       regions.add(p);
1106     }
1107   }
1108 
1109   private int getRegionReplication(HRegionInfo r) throws IOException {
1110     if (tableStateManager != null) {
1111       HTableDescriptor htd = ((MasterServices)server).getTableDescriptors().get(r.getTable());
1112       if (htd != null) {
1113         return htd.getRegionReplication();
1114       }
1115     }
1116     return 1;
1117   }
1118 
1119   /**
1120    * At cluster clean re/start, mark all user regions closed except those of tables
1121    * that are excluded, such as disabled/disabling/enabling tables. All user regions
1122    * and their previous locations are returned.
1123    */
1124   synchronized Map<HRegionInfo, ServerName> closeAllUserRegions(Set<TableName> excludedTables) {
1125     boolean noExcludeTables = excludedTables == null || excludedTables.isEmpty();
1126     Set<HRegionInfo> toBeClosed = new HashSet<HRegionInfo>(regionStates.size());
1127     for(RegionState state: regionStates.values()) {
1128       HRegionInfo hri = state.getRegion();
1129       if (state.isSplit() || hri.isSplit()) {
1130         continue;
1131       }
1132       TableName tableName = hri.getTable();
1133       if (!TableName.META_TABLE_NAME.equals(tableName)
1134           && (noExcludeTables || !excludedTables.contains(tableName))) {
1135         toBeClosed.add(hri);
1136       }
1137     }
1138     Map<HRegionInfo, ServerName> allUserRegions =
1139       new HashMap<HRegionInfo, ServerName>(toBeClosed.size());
1140     for (HRegionInfo hri: toBeClosed) {
1141       RegionState regionState = updateRegionState(hri, State.CLOSED);
1142       allUserRegions.put(hri, regionState.getServerName());
1143     }
1144     return allUserRegions;
1145   }
1146 
1147   /**
1148    * Compute the average load across all region servers.
1149    * Currently, this uses a very naive computation - just uses the number of
1150    * regions being served, ignoring stats about number of requests.
1151    * @return the average load
1152    */
1153   protected synchronized double getAverageLoad() {
1154     int numServers = 0, totalLoad = 0;
1155     for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1156       Set<HRegionInfo> regions = e.getValue();
1157       ServerName serverName = e.getKey();
1158       int regionCount = regions.size();
1159       if (serverManager.isServerOnline(serverName)) {
1160         totalLoad += regionCount;
1161         numServers++;
1162       }
1163     }
1164     if (numServers > 1) {
1165       // The master region server holds only a couple regions.
1166       // Don't consider this server in calculating the average load
1167       // if there are other region servers to avoid possible confusion.
1168       Set<HRegionInfo> hris = serverHoldings.get(server.getServerName());
1169       if (hris != null) {
1170         totalLoad -= hris.size();
1171         numServers--;
1172       }
1173     }
1174     return numServers == 0 ? 0.0 :
1175       (double)totalLoad / (double)numServers;
1176   }
1177 
1178   /**
1179    * This is an EXPENSIVE clone.  Cloning though is the safest thing to do.
1180    * Can't let out original since it can change and at least the load balancer
1181    * wants to iterate this exported list.  We need to synchronize on regions
1182    * since all access to this.servers is under a lock on this.regions.
1183    *
1184    * @return A clone of current assignments by table.
1185    */
1186   protected Map<TableName, Map<ServerName, List<HRegionInfo>>>
1187       getAssignmentsByTable() {
1188     Map<TableName, Map<ServerName, List<HRegionInfo>>> result =
1189       new HashMap<TableName, Map<ServerName,List<HRegionInfo>>>();
1190     synchronized (this) {
1191       if (!server.getConfiguration().getBoolean(
1192             HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false)) {
1193         Map<ServerName, List<HRegionInfo>> svrToRegions =
1194           new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1195         for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1196           svrToRegions.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
1197         }
1198         result.put(TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME), svrToRegions);
1199       } else {
1200         for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1201           for (HRegionInfo hri: e.getValue()) {
1202             if (hri.isMetaRegion()) continue;
1203             TableName tablename = hri.getTable();
1204             Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
1205             if (svrToRegions == null) {
1206               svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1207               result.put(tablename, svrToRegions);
1208             }
1209             List<HRegionInfo> regions = svrToRegions.get(e.getKey());
1210             if (regions == null) {
1211               regions = new ArrayList<HRegionInfo>();
1212               svrToRegions.put(e.getKey(), regions);
1213             }
1214             regions.add(hri);
1215           }
1216         }
1217       }
1218     }
1219 
1220     Map<ServerName, ServerLoad>
1221       onlineSvrs = serverManager.getOnlineServers();
1222     // Take care of servers w/o assignments, and remove servers in draining mode
1223     List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
1224     for (Map<ServerName, List<HRegionInfo>> map: result.values()) {
1225       for (ServerName svr: onlineSvrs.keySet()) {
1226         if (!map.containsKey(svr)) {
1227           map.put(svr, new ArrayList<HRegionInfo>());
1228         }
1229       }
1230       map.keySet().removeAll(drainingServers);
1231     }
1232     return result;
1233   }
1234 
1235   protected RegionState getRegionState(final HRegionInfo hri) {
1236     return getRegionState(hri.getEncodedName());
1237   }
1238 
1239   /**
1240    * Returns a clone of region assignments per server
1241    * @return a Map of ServerName to a List of HRegionInfo's
1242    */
1243   protected synchronized Map<ServerName, List<HRegionInfo>> getRegionAssignmentsByServer() {
1244     Map<ServerName, List<HRegionInfo>> regionsByServer =
1245         new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
1246     for (Map.Entry<ServerName, Set<HRegionInfo>> e: serverHoldings.entrySet()) {
1247       regionsByServer.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
1248     }
1249     return regionsByServer;
1250   }
1251 
1252   protected synchronized RegionState getRegionState(final String encodedName) {
1253     return regionStates.get(encodedName);
1254   }
1255 
1256   /**
1257    * Get the HRegionInfo from cache, if not there, from the hbase:meta table
1258    * @param  regionName
1259    * @return HRegionInfo for the region
1260    */
1261   @SuppressWarnings("deprecation")
1262   protected HRegionInfo getRegionInfo(final byte [] regionName) {
1263     String encodedName = HRegionInfo.encodeRegionName(regionName);
1264     RegionState regionState = getRegionState(encodedName);
1265     if (regionState != null) {
1266       return regionState.getRegion();
1267     }
1268 
1269     try {
1270       Pair<HRegionInfo, ServerName> p =
1271         MetaTableAccessor.getRegion(server.getConnection(), regionName);
1272       HRegionInfo hri = p == null ? null : p.getFirst();
1273       if (hri != null) {
1274         createRegionState(hri);
1275       }
1276       return hri;
1277     } catch (IOException e) {
1278       server.abort("Aborting because error occurred while reading "
1279         + Bytes.toStringBinary(regionName) + " from hbase:meta", e);
1280       return null;
1281     }
1282   }
1283 
1284   static boolean isOneOfStates(RegionState regionState, State... states) {
1285     State s = regionState != null ? regionState.getState() : null;
1286     for (State state: states) {
1287       if (s == state) return true;
1288     }
1289     return false;
1290   }
1291 
1292   /**
1293    * Update a region state. It will be put in transition if not already there.
1294    */
1295   private RegionState updateRegionState(final HRegionInfo hri,
1296       final State state, final ServerName serverName, long openSeqNum) {
1297     if (state == State.FAILED_CLOSE || state == State.FAILED_OPEN) {
1298       LOG.warn("Failed to open/close " + hri.getShortNameToLog()
1299         + " on " + serverName + ", set to " + state);
1300     }
1301 
1302     String encodedName = hri.getEncodedName();
1303     RegionState regionState = new RegionState(
1304       hri, state, System.currentTimeMillis(), serverName);
1305     RegionState oldState = getRegionState(encodedName);
1306     if (!regionState.equals(oldState)) {
1307       LOG.info("Transition " + oldState + " to " + regionState);
1308       // Persist region state before updating in-memory info, if needed
1309       regionStateStore.updateRegionState(openSeqNum, regionState, oldState);
1310     }
1311 
1312     synchronized (this) {
1313       RegionState oldRegionState = regionsInTransition.put(encodedName, regionState);
1314       // When region transform old region state to new region state,
1315       // accumulate the RIT duration to new region state.
1316       if (oldRegionState != null) {
1317         regionState.updateRitDuration(oldRegionState.getStamp());
1318       }
1319       putRegionState(regionState);
1320 
1321       // For these states, region should be properly closed.
1322       // There should be no log splitting issue.
1323       if ((state == State.CLOSED || state == State.MERGED
1324           || state == State.SPLIT) && lastAssignments.containsKey(encodedName)) {
1325         ServerName last = lastAssignments.get(encodedName);
1326         if (last.equals(serverName)) {
1327           lastAssignments.remove(encodedName);
1328         } else {
1329           LOG.warn(encodedName + " moved to " + state + " on "
1330             + serverName + ", expected " + last);
1331         }
1332       }
1333 
1334       // Once a region is opened, record its last assignment right away.
1335       if (serverName != null && state == State.OPEN) {
1336         ServerName last = lastAssignments.get(encodedName);
1337         if (!serverName.equals(last)) {
1338           lastAssignments.put(encodedName, serverName);
1339           if (last != null && isServerDeadAndNotProcessed(last)) {
1340             LOG.warn(encodedName + " moved to " + serverName
1341               + ", while it's previous host " + last
1342               + " is dead but not processed yet");
1343           }
1344         }
1345       }
1346 
1347       // notify the change
1348       this.notifyAll();
1349     }
1350     return regionState;
1351   }
1352 }