View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.util.ArrayList;
22  import java.util.Comparator;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.NavigableSet;
26  import java.util.TreeMap;
27  import java.util.TreeSet;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.util.DNS;
30  
31  /**
32   * Data structure to describe the distribution of HDFS blocks among hosts.
33   *
34   * Adding erroneous data will be ignored silently.
35   */
36  @InterfaceAudience.Private
37  public class HDFSBlocksDistribution {
38    private Map<String,HostAndWeight> hostAndWeights = null;
39    private long uniqueBlocksTotalWeight = 0;
40  
41    /**
42     * Stores the hostname and weight for that hostname.
43     *
44     * This is used when determining the physical locations of the blocks making
45     * up a region.
46     *
47     * To make a prioritized list of the hosts holding the most data of a region,
48     * this class is used to count the total weight for each host.  The weight is
49     * currently just the size of the file.
50     */
51    public static class HostAndWeight {
52  
53      private String host;
54      private long weight;
55  
56      /**
57       * Constructor
58       * @param host the host name
59       * @param weight the weight
60       */
61      public HostAndWeight(String host, long weight) {
62        this.host = host;
63        this.weight = weight;
64      }
65  
66      /**
67       * add weight
68       * @param weight the weight
69       */
70      public void addWeight(long weight) {
71        this.weight += weight;
72      }
73  
74      /**
75       * @return the host name
76       */
77      public String getHost() {
78        return host;
79      }
80  
81      /**
82       * @return the weight
83       */
84      public long getWeight() {
85        return weight;
86      }
87  
88      /**
89       * comparator used to sort hosts based on weight
90       */
91      public static class WeightComparator implements Comparator<HostAndWeight> {
92        @Override
93        public int compare(HostAndWeight l, HostAndWeight r) {
94          if(l.getWeight() == r.getWeight()) {
95            return l.getHost().compareTo(r.getHost());
96          }
97          return l.getWeight() < r.getWeight() ? -1 : 1;
98        }
99      }
100   }
101 
102   /**
103    * Constructor
104    */
105   public HDFSBlocksDistribution() {
106     this.hostAndWeights =
107       new TreeMap<String,HostAndWeight>();
108   }
109 
110   /**
111    * @see java.lang.Object#toString()
112    */
113   @Override
114   public synchronized String toString() {
115     return "number of unique hosts in the distribution=" +
116       this.hostAndWeights.size();
117   }
118 
119   /**
120    * add some weight to a list of hosts, update the value of unique block weight
121    * @param hosts the list of the host
122    * @param weight the weight
123    */
124   public void addHostsAndBlockWeight(String[] hosts, long weight) {
125     if (hosts == null || hosts.length == 0) {
126       // erroneous data
127       return;
128     }
129 
130     addUniqueWeight(weight);
131     for (String hostname : hosts) {
132       addHostAndBlockWeight(hostname, weight);
133     }
134   }
135 
136   /**
137    * add some weight to the total unique weight
138    * @param weight the weight
139    */
140   private void addUniqueWeight(long weight) {
141     uniqueBlocksTotalWeight += weight;
142   }
143 
144 
145   /**
146    * add some weight to a specific host
147    * @param host the host name
148    * @param weight the weight
149    */
150   private void addHostAndBlockWeight(String host, long weight) {
151     if (host == null) {
152       // erroneous data
153       return;
154     }
155 
156     HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
157     if(hostAndWeight == null) {
158       hostAndWeight = new HostAndWeight(host, weight);
159       this.hostAndWeights.put(host, hostAndWeight);
160     } else {
161       hostAndWeight.addWeight(weight);
162     }
163   }
164 
165   /**
166    * @return the hosts and their weights
167    */
168   public Map<String,HostAndWeight> getHostAndWeights() {
169     return this.hostAndWeights;
170   }
171 
172   /**
173    * return the weight for a specific host, that will be the total bytes of all
174    * blocks on the host
175    * @param host the host name
176    * @return the weight of the given host
177    */
178   public long getWeight(String host) {
179     long weight = 0;
180     if (host != null) {
181       HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
182       if(hostAndWeight != null) {
183         weight = hostAndWeight.getWeight();
184       }
185     }
186     return weight;
187   }
188 
189   /**
190    * @return the sum of all unique blocks' weight
191    */
192   public long getUniqueBlocksTotalWeight() {
193     return uniqueBlocksTotalWeight;
194   }
195 
196   /**
197    * return the locality index of a given host
198    * @param host the host name
199    * @return the locality index of the given host
200    */
201   public float getBlockLocalityIndex(String host) {
202     float localityIndex = 0;
203     HostAndWeight hostAndWeight = this.hostAndWeights.get(host);
204     // Compatible with local mode, see HBASE-24569
205     if (hostAndWeight == null) {
206       String currentHost = "";
207       try {
208         currentHost = DNS.getDefaultHost("default", "default");
209       } catch (Exception e) {
210         // Just ignore, it's ok, avoid too many log info
211       }
212       if (host.equals(currentHost)) {
213         hostAndWeight = this.hostAndWeights.get(HConstants.LOCALHOST);
214       }
215     }
216     if (hostAndWeight != null && uniqueBlocksTotalWeight != 0) {
217       localityIndex=(float)hostAndWeight.weight/(float)uniqueBlocksTotalWeight;
218     }
219     return localityIndex;
220   }
221 
222 
223   /**
224    * This will add the distribution from input to this object
225    * @param otherBlocksDistribution the other hdfs blocks distribution
226    */
227   public void add(HDFSBlocksDistribution otherBlocksDistribution) {
228     Map<String,HostAndWeight> otherHostAndWeights =
229       otherBlocksDistribution.getHostAndWeights();
230     for (Map.Entry<String, HostAndWeight> otherHostAndWeight:
231       otherHostAndWeights.entrySet()) {
232       addHostAndBlockWeight(otherHostAndWeight.getValue().host,
233         otherHostAndWeight.getValue().weight);
234     }
235     addUniqueWeight(otherBlocksDistribution.getUniqueBlocksTotalWeight());
236   }
237 
238   /**
239    * return the sorted list of hosts in terms of their weights
240    */
241   public List<String> getTopHosts() {
242     HostAndWeight[] hostAndWeights = getTopHostsWithWeights();
243     List<String> topHosts = new ArrayList<String>(hostAndWeights.length);
244     for(HostAndWeight haw : hostAndWeights) {
245       topHosts.add(haw.getHost());
246     }
247     return topHosts;
248   }
249 
250   /**
251    * return the sorted list of hosts in terms of their weights
252    */
253   public HostAndWeight[] getTopHostsWithWeights() {
254     NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
255       new HostAndWeight.WeightComparator());
256     orderedHosts.addAll(this.hostAndWeights.values());
257     return orderedHosts.descendingSet().toArray(new HostAndWeight[orderedHosts.size()]);
258   }
259 
260 }