1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import com.google.protobuf.ServiceException;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.client.ClusterConnection;
27 import org.apache.hadoop.hbase.client.Connection;
28 import org.apache.hadoop.hbase.client.ConnectionFactory;
29 import org.apache.hadoop.hbase.client.Delete;
30 import org.apache.hadoop.hbase.client.Get;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.Mutation;
33 import org.apache.hadoop.hbase.client.Put;
34 import org.apache.hadoop.hbase.client.RegionLocator;
35 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.ResultScanner;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.client.Table;
40 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
41 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
42 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
43 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46 import org.apache.hadoop.hbase.util.Pair;
47 import org.apache.hadoop.hbase.util.PairOfSameType;
48 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
49 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
50
51 import java.io.IOException;
52 import java.io.InterruptedIOException;
53 import java.util.ArrayList;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.NavigableMap;
57 import java.util.Set;
58 import java.util.SortedMap;
59 import java.util.TreeMap;
60 import java.util.regex.Matcher;
61 import java.util.regex.Pattern;
62
63
64
65
66
67
68
69
70
71
72 @InterfaceAudience.Private
73 public class MetaTableAccessor {
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111 private static final Log LOG = LogFactory.getLog(MetaTableAccessor.class);
112
113 static final byte [] META_REGION_PREFIX;
114 static {
115
116
117 int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
118 META_REGION_PREFIX = new byte [len];
119 System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
120 META_REGION_PREFIX, 0, len);
121 }
122
123
124 protected static final char META_REPLICA_ID_DELIMITER = '_';
125
126
127 private static final Pattern SERVER_COLUMN_PATTERN
128 = Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
129
130
131
132
133
134
135
136
137
138
139 public static List<Result> fullScanOfMeta(Connection connection)
140 throws IOException {
141 CollectAllVisitor v = new CollectAllVisitor();
142 fullScan(connection, v, null);
143 return v.getResults();
144 }
145
146
147
148
149
150
151
152 public static void fullScan(Connection connection,
153 final Visitor visitor)
154 throws IOException {
155 fullScan(connection, visitor, null);
156 }
157
158
159
160
161
162
163
164 public static List<Result> fullScan(Connection connection)
165 throws IOException {
166 CollectAllVisitor v = new CollectAllVisitor();
167 fullScan(connection, v, null);
168 return v.getResults();
169 }
170
171
172
173
174
175
176
177 static Table getMetaHTable(final Connection connection)
178 throws IOException {
179
180 if (connection == null) {
181 throw new NullPointerException("No connection");
182 } else if (connection.isClosed()) {
183 throw new IOException("connection is closed");
184 }
185
186
187
188
189
190
191
192
193 if (connection instanceof ClusterConnection) {
194 if (((ClusterConnection) connection).isManaged()) {
195 return new HTable(TableName.META_TABLE_NAME, (ClusterConnection) connection);
196 }
197 }
198 return connection.getTable(TableName.META_TABLE_NAME);
199 }
200
201
202
203
204
205
206 private static Result get(final Table t, final Get g) throws IOException {
207 try {
208 return t.get(g);
209 } finally {
210 t.close();
211 }
212 }
213
214
215
216
217
218
219
220
221
222 @Deprecated
223 public static Pair<HRegionInfo, ServerName> getRegion(Connection connection, byte [] regionName)
224 throws IOException {
225 HRegionLocation location = getRegionLocation(connection, regionName);
226 return location == null
227 ? null
228 : new Pair<HRegionInfo, ServerName>(location.getRegionInfo(), location.getServerName());
229 }
230
231
232
233
234
235
236
237
238 public static HRegionLocation getRegionLocation(Connection connection,
239 byte[] regionName) throws IOException {
240 byte[] row = regionName;
241 HRegionInfo parsedInfo = null;
242 try {
243 parsedInfo = parseRegionInfoFromRegionName(regionName);
244 row = getMetaKeyForRegion(parsedInfo);
245 } catch (Exception parseEx) {
246
247 }
248 Get get = new Get(row);
249 get.addFamily(HConstants.CATALOG_FAMILY);
250 Result r = get(getMetaHTable(connection), get);
251 RegionLocations locations = getRegionLocations(r);
252 return locations == null
253 ? null
254 : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
255 }
256
257
258
259
260
261
262
263
264 public static HRegionLocation getRegionLocation(Connection connection,
265 HRegionInfo regionInfo) throws IOException {
266 byte[] row = getMetaKeyForRegion(regionInfo);
267 Get get = new Get(row);
268 get.addFamily(HConstants.CATALOG_FAMILY);
269 Result r = get(getMetaHTable(connection), get);
270 return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
271 }
272
273
274 public static byte[] getMetaKeyForRegion(HRegionInfo regionInfo) {
275 return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
276 }
277
278
279
280
281
282 protected static HRegionInfo parseRegionInfoFromRegionName(byte[] regionName)
283 throws IOException {
284 byte[][] fields = HRegionInfo.parseRegionName(regionName);
285 long regionId = Long.parseLong(Bytes.toString(fields[2]));
286 int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
287 return new HRegionInfo(
288 TableName.valueOf(fields[0]), fields[1], fields[1], false, regionId, replicaId);
289 }
290
291
292
293
294
295
296
297
298 public static Result getRegionResult(Connection connection,
299 byte[] regionName) throws IOException {
300 Get get = new Get(regionName);
301 get.addFamily(HConstants.CATALOG_FAMILY);
302 return get(getMetaHTable(connection), get);
303 }
304
305
306
307
308
309
310 public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
311 Connection connection, byte[] regionName) throws IOException {
312 Result result = getRegionResult(connection, regionName);
313 HRegionInfo mergeA = getHRegionInfo(result, HConstants.MERGEA_QUALIFIER);
314 HRegionInfo mergeB = getHRegionInfo(result, HConstants.MERGEB_QUALIFIER);
315 if (mergeA == null && mergeB == null) {
316 return null;
317 }
318 return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
319 }
320
321
322
323
324
325
326
327
328
329 public static boolean tableExists(Connection connection,
330 final TableName tableName)
331 throws IOException {
332 if (tableName.equals(TableName.META_TABLE_NAME)) {
333
334 return true;
335 }
336
337 CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
338 private HRegionInfo current = null;
339
340 @Override
341 public boolean visit(Result r) throws IOException {
342 RegionLocations locations = getRegionLocations(r);
343 if (locations == null || locations.getRegionLocation().getRegionInfo() == null) {
344 LOG.warn("No serialized HRegionInfo in " + r);
345 return true;
346 }
347 this.current = locations.getRegionLocation().getRegionInfo();
348 if (this.current == null) {
349 LOG.warn("No serialized HRegionInfo in " + r);
350 return true;
351 }
352 if (!isInsideTable(this.current, tableName)) return false;
353
354 super.visit(r);
355
356 return false;
357 }
358
359 @Override
360 void add(Result r) {
361
362 this.results.add(this.current);
363 }
364 };
365 fullScan(connection, visitor, getTableStartRowForMeta(tableName));
366
367 return visitor.getResults().size() >= 1;
368 }
369
370
371
372
373
374
375
376
377
378 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw,
379 Connection connection, TableName tableName)
380 throws IOException {
381 return getTableRegions(zkw, connection, tableName, false);
382 }
383
384
385
386
387
388
389
390
391
392
393
394 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
395 TableName tableName, final boolean excludeOfflinedSplitParents) throws IOException {
396 return getTableRegions(zkw, connection, tableName, excludeOfflinedSplitParents, false);
397 }
398
399
400
401
402
403
404
405
406
407
408
409 public static List<HRegionInfo> getTableRegions(ZooKeeperWatcher zkw, Connection connection,
410 TableName tableName, final boolean excludeOfflinedSplitParents,
411 final boolean excludeReplicaRegions) throws IOException {
412 List<Pair<HRegionInfo, ServerName>> result = null;
413 result = getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
414 excludeReplicaRegions);
415 return getListOfHRegionInfos(result);
416 }
417
418 static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
419 if (pairs == null || pairs.isEmpty()) return null;
420 List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
421 for (Pair<HRegionInfo, ServerName> pair: pairs) {
422 result.add(pair.getFirst());
423 }
424 return result;
425 }
426
427
428
429
430
431
432
433 static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
434 return tableName.equals(current.getTable());
435 }
436
437
438
439
440
441
442 static byte [] getTableStartRowForMeta(TableName tableName) {
443 byte [] startRow = new byte[tableName.getName().length + 2];
444 System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
445 startRow[startRow.length - 2] = HConstants.DELIMITER;
446 startRow[startRow.length - 1] = HConstants.DELIMITER;
447 return startRow;
448 }
449
450
451
452
453
454
455
456
457
458
459 public static Scan getScanForTableName(TableName tableName) {
460 String strName = tableName.getNameAsString();
461
462 byte[] startKey = Bytes.toBytes(strName + ",,");
463
464 byte[] stopKey = Bytes.toBytes(strName + " ,,");
465
466 Scan scan = new Scan(startKey);
467 scan.setStopRow(stopKey);
468 return scan;
469 }
470
471
472
473
474
475
476
477
478 public static List<Pair<HRegionInfo, ServerName>>
479 getTableRegionsAndLocations(ZooKeeperWatcher zkw,
480 Connection connection, TableName tableName)
481 throws IOException {
482 return getTableRegionsAndLocations(zkw, connection, tableName, true);
483 }
484
485
486
487
488
489
490
491
492 public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
493 ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
494 final boolean excludeOfflinedSplitParents) throws IOException {
495 return getTableRegionsAndLocations(zkw, connection, tableName, excludeOfflinedSplitParents,
496 false);
497 }
498
499
500
501
502
503
504
505
506
507 public static List<Pair<HRegionInfo, ServerName>> getTableRegionsAndLocations(
508 ZooKeeperWatcher zkw, Connection connection, final TableName tableName,
509 final boolean excludeOfflinedSplitParents, final boolean excludeReplicaRegions)
510 throws IOException {
511 if (tableName.equals(TableName.META_TABLE_NAME)) {
512
513 ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw);
514 List<Pair<HRegionInfo, ServerName>> list =
515 new ArrayList<Pair<HRegionInfo, ServerName>>();
516 list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
517 serverName));
518 return list;
519 }
520
521 CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
522 new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
523 private RegionLocations current = null;
524
525 @Override
526 public boolean visit(Result r) throws IOException {
527 current = getRegionLocations(r);
528 if (current == null || current.getRegionLocation().getRegionInfo() == null) {
529 LOG.warn("No serialized HRegionInfo in " + r);
530 return true;
531 }
532 HRegionInfo hri = current.getRegionLocation().getRegionInfo();
533 if (!isInsideTable(hri, tableName)) return false;
534 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
535
536 return super.visit(r);
537 }
538
539 @Override
540 void add(Result r) {
541 if (current == null) {
542 return;
543 }
544 for (HRegionLocation loc : current.getRegionLocations()) {
545 if (loc != null) {
546
547 if (excludeReplicaRegions
548 && !RegionReplicaUtil.isDefaultReplica(loc.getRegionInfo())) {
549 continue;
550 }
551 this.results.add(new Pair<HRegionInfo, ServerName>(
552 loc.getRegionInfo(), loc.getServerName()));
553 }
554 }
555 }
556 };
557 fullScan(connection, visitor, getTableStartRowForMeta(tableName));
558 return visitor.getResults();
559 }
560
561
562
563
564
565
566
567
568 public static NavigableMap<HRegionInfo, Result>
569 getServerUserRegions(Connection connection, final ServerName serverName)
570 throws IOException {
571 final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
572
573
574 CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
575 @Override
576 void add(Result r) {
577 if (r == null || r.isEmpty()) return;
578 RegionLocations locations = getRegionLocations(r);
579 if (locations == null) return;
580 for (HRegionLocation loc : locations.getRegionLocations()) {
581 if (loc != null) {
582 if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
583 hris.put(loc.getRegionInfo(), r);
584 }
585 }
586 }
587 }
588 };
589 fullScan(connection, v);
590 return hris;
591 }
592
593 public static void fullScanMetaAndPrint(Connection connection)
594 throws IOException {
595 Visitor v = new Visitor() {
596 @Override
597 public boolean visit(Result r) throws IOException {
598 if (r == null || r.isEmpty()) return true;
599 LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
600 RegionLocations locations = getRegionLocations(r);
601 if (locations == null) return true;
602 for (HRegionLocation loc : locations.getRegionLocations()) {
603 if (loc != null) {
604 LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
605 }
606 }
607 return true;
608 }
609 };
610 fullScan(connection, v);
611 }
612
613
614
615
616
617
618
619
620
621
622 public static void fullScan(Connection connection,
623 final Visitor visitor, final byte [] startrow)
624 throws IOException {
625 Scan scan = new Scan();
626 if (startrow != null) scan.setStartRow(startrow);
627 if (startrow == null) {
628 int caching = connection.getConfiguration()
629 .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
630 scan.setCaching(caching);
631 }
632 scan.addFamily(HConstants.CATALOG_FAMILY);
633 Table metaTable = getMetaHTable(connection);
634 ResultScanner scanner = null;
635 try {
636 scanner = metaTable.getScanner(scan);
637 Result data;
638 while((data = scanner.next()) != null) {
639 if (data.isEmpty()) continue;
640
641 if (!visitor.visit(data)) break;
642 }
643 } finally {
644 if (scanner != null) scanner.close();
645 metaTable.close();
646 }
647 }
648
649
650
651
652
653 protected static byte[] getFamily() {
654 return HConstants.CATALOG_FAMILY;
655 }
656
657
658
659
660
661 protected static byte[] getRegionInfoColumn() {
662 return HConstants.REGIONINFO_QUALIFIER;
663 }
664
665
666
667
668
669
670 public static byte[] getServerColumn(int replicaId) {
671 return replicaId == 0
672 ? HConstants.SERVER_QUALIFIER
673 : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
674 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
675 }
676
677
678
679
680
681
682 public static byte[] getStartCodeColumn(int replicaId) {
683 return replicaId == 0
684 ? HConstants.STARTCODE_QUALIFIER
685 : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
686 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
687 }
688
689
690
691
692
693
694 public static byte[] getSeqNumColumn(int replicaId) {
695 return replicaId == 0
696 ? HConstants.SEQNUM_QUALIFIER
697 : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
698 + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
699 }
700
701
702
703
704
705
706
707 static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
708 String serverStr = Bytes.toString(serverColumn);
709
710 Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
711 if (matcher.matches() && matcher.groupCount() > 0) {
712 String group = matcher.group(1);
713 if (group != null && group.length() > 0) {
714 return Integer.parseInt(group.substring(1), 16);
715 } else {
716 return 0;
717 }
718 }
719 return -1;
720 }
721
722
723
724
725
726
727 private static ServerName getServerName(final Result r, final int replicaId) {
728 byte[] serverColumn = getServerColumn(replicaId);
729 Cell cell = r.getColumnLatestCell(getFamily(), serverColumn);
730 if (cell == null || cell.getValueLength() == 0) return null;
731 String hostAndPort = Bytes.toString(
732 cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
733 byte[] startcodeColumn = getStartCodeColumn(replicaId);
734 cell = r.getColumnLatestCell(getFamily(), startcodeColumn);
735 if (cell == null || cell.getValueLength() == 0) return null;
736 return ServerName.valueOf(hostAndPort,
737 Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
738 }
739
740
741
742
743
744
745
746 private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
747 Cell cell = r.getColumnLatestCell(getFamily(), getSeqNumColumn(replicaId));
748 if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
749 return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
750 }
751
752
753
754
755
756
757 public static RegionLocations getRegionLocations(final Result r) {
758 if (r == null) return null;
759 HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
760 if (regionInfo == null) return null;
761
762 List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
763 NavigableMap<byte[],NavigableMap<byte[],byte[]>> familyMap = r.getNoVersionMap();
764
765 locations.add(getRegionLocation(r, regionInfo, regionInfo.getReplicaId()));
766
767 NavigableMap<byte[], byte[]> infoMap = familyMap.get(getFamily());
768 if (infoMap == null) return new RegionLocations(locations);
769
770
771 int replicaId = 0;
772 byte[] serverColumn = getServerColumn(replicaId);
773 SortedMap<byte[], byte[]> serverMap = null;
774 serverMap = infoMap.tailMap(serverColumn, false);
775
776 if (serverMap.isEmpty()) return new RegionLocations(locations);
777
778 for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) {
779 replicaId = parseReplicaIdFromServerColumn(entry.getKey());
780 if (replicaId < 0) {
781 break;
782 }
783 HRegionLocation location = getRegionLocation(r, regionInfo, replicaId);
784
785
786 if (location == null || location.getServerName() == null) {
787 locations.add(null);
788 } else {
789 locations.add(location);
790 }
791 }
792
793 return new RegionLocations(locations);
794 }
795
796
797
798
799
800
801
802
803
804
805 private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo,
806 final int replicaId) {
807 ServerName serverName = getServerName(r, replicaId);
808 long seqNum = getSeqNumDuringOpen(r, replicaId);
809 HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
810 return new HRegionLocation(replicaInfo, serverName, seqNum);
811 }
812
813
814
815
816
817
818
819
820 public static HRegionInfo getHRegionInfo(Result data) {
821 return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
822 }
823
824
825
826
827
828
829
830
831 private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
832 Cell cell = r.getColumnLatestCell(getFamily(), qualifier);
833 if (cell == null) return null;
834 return HRegionInfo.parseFromOrNull(cell.getValueArray(),
835 cell.getValueOffset(), cell.getValueLength());
836 }
837
838
839
840
841
842
843
844
845
846 public static PairOfSameType<HRegionInfo> getDaughterRegionsFromParent(
847 final Connection connection, HRegionInfo parent) throws IOException {
848 Result parentResult = getRegionResult(connection, parent.getRegionName());
849 return getDaughterRegions(parentResult);
850 }
851
852
853
854
855
856
857
858
859 public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) {
860 HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
861 HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
862
863 return new PairOfSameType<HRegionInfo>(splitA, splitB);
864 }
865
866
867
868
869
870
871
872
873 public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) {
874 HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
875 HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
876
877 return new PairOfSameType<HRegionInfo>(mergeA, mergeB);
878 }
879
880
881
882
883 public interface Visitor {
884
885
886
887
888
889
890 boolean visit(final Result r) throws IOException;
891 }
892
893
894
895
896 static abstract class CollectingVisitor<T> implements Visitor {
897 final List<T> results = new ArrayList<T>();
898 @Override
899 public boolean visit(Result r) throws IOException {
900 if (r == null || r.isEmpty()) return true;
901 add(r);
902 return true;
903 }
904
905 abstract void add(Result r);
906
907
908
909
910
911 List<T> getResults() {
912 return this.results;
913 }
914 }
915
916
917
918
919 static class CollectAllVisitor extends CollectingVisitor<Result> {
920 @Override
921 void add(Result r) {
922 this.results.add(r);
923 }
924 }
925
926
927
928
929
930
931
932
933 @Deprecated
934 public static int getRegionCount(final Configuration c, final String tableName)
935 throws IOException {
936 return getRegionCount(c, TableName.valueOf(tableName));
937 }
938
939
940
941
942
943
944
945
946 public static int getRegionCount(final Configuration c, final TableName tableName)
947 throws IOException {
948 try (Connection connection = ConnectionFactory.createConnection(c)) {
949 return getRegionCount(connection, tableName);
950 }
951 }
952
953
954
955
956
957
958
959
960 public static int getRegionCount(final Connection connection, final TableName tableName)
961 throws IOException {
962 try (RegionLocator locator = connection.getRegionLocator(tableName)) {
963 List<HRegionLocation> locations = locator.getAllRegionLocations();
964 return locations == null? 0: locations.size();
965 }
966 }
967
968
969
970
971
972
973
974
975 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
976 throws IOException {
977 return makePutFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
978 }
979
980
981
982 public static Put makePutFromRegionInfo(HRegionInfo regionInfo, long ts)
983 throws IOException {
984 Put put = new Put(regionInfo.getRegionName(), ts);
985 addRegionInfo(put, regionInfo);
986 return put;
987 }
988
989
990
991
992
993 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
994 return makeDeleteFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
995 }
996
997
998
999
1000
1001 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo, long ts) {
1002 if (regionInfo == null) {
1003 throw new IllegalArgumentException("Can't make a delete for null region");
1004 }
1005 Delete delete = new Delete(regionInfo.getRegionName(), ts);
1006 return delete;
1007 }
1008
1009
1010
1011
1012 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
1013 if (splitA != null) {
1014 put.addImmutable(
1015 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
1016 }
1017 if (splitB != null) {
1018 put.addImmutable(
1019 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
1020 }
1021 return put;
1022 }
1023
1024
1025
1026
1027
1028
1029
1030 static void putToMetaTable(final Connection connection, final Put p)
1031 throws IOException {
1032 put(getMetaHTable(connection), p);
1033 }
1034
1035
1036
1037
1038
1039
1040 private static void put(final Table t, final Put p) throws IOException {
1041 try {
1042 t.put(p);
1043 } finally {
1044 t.close();
1045 }
1046 }
1047
1048
1049
1050
1051
1052
1053
1054 public static void putsToMetaTable(final Connection connection, final List<Put> ps)
1055 throws IOException {
1056 Table t = getMetaHTable(connection);
1057 try {
1058 t.put(ps);
1059 } finally {
1060 t.close();
1061 }
1062 }
1063
1064
1065
1066
1067
1068
1069
1070 static void deleteFromMetaTable(final Connection connection, final Delete d)
1071 throws IOException {
1072 List<Delete> dels = new ArrayList<Delete>(1);
1073 dels.add(d);
1074 deleteFromMetaTable(connection, dels);
1075 }
1076
1077
1078
1079
1080
1081
1082
1083 public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
1084 throws IOException {
1085 Table t = getMetaHTable(connection);
1086 try {
1087 t.delete(deletes);
1088 } finally {
1089 t.close();
1090 }
1091 }
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101 public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows,
1102 int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection)
1103 throws IOException {
1104 int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove;
1105 for (byte[] row : metaRows) {
1106 Delete deleteReplicaLocations = new Delete(row);
1107 for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) {
1108 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1109 getServerColumn(i));
1110 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1111 getSeqNumColumn(i));
1112 deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY,
1113 getStartCodeColumn(i));
1114 }
1115 deleteFromMetaTable(connection, deleteReplicaLocations);
1116 }
1117 }
1118
1119
1120
1121
1122
1123
1124
1125 public static void mutateMetaTable(final Connection connection,
1126 final List<Mutation> mutations)
1127 throws IOException {
1128 Table t = getMetaHTable(connection);
1129 try {
1130 t.batch(mutations);
1131 } catch (InterruptedException e) {
1132 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
1133 ie.initCause(e);
1134 throw ie;
1135 } finally {
1136 t.close();
1137 }
1138 }
1139
1140
1141
1142
1143
1144
1145
1146 public static void addRegionToMeta(Connection connection,
1147 HRegionInfo regionInfo)
1148 throws IOException {
1149 putToMetaTable(connection, makePutFromRegionInfo(regionInfo));
1150 LOG.info("Added " + regionInfo.getRegionNameAsString());
1151 }
1152
1153
1154
1155
1156
1157
1158
1159
1160 public static void addRegionToMeta(Table meta, HRegionInfo regionInfo) throws IOException {
1161 addRegionToMeta(meta, regionInfo, null, null);
1162 }
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176 public static void addRegionToMeta(Table meta, HRegionInfo regionInfo,
1177 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
1178 Put put = makePutFromRegionInfo(regionInfo);
1179 addDaughtersToPut(put, splitA, splitB);
1180 meta.put(put);
1181 if (LOG.isDebugEnabled()) {
1182 LOG.debug("Added " + regionInfo.getRegionNameAsString());
1183 }
1184 }
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198 public static void addRegionToMeta(Connection connection, HRegionInfo regionInfo,
1199 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
1200 Table meta = getMetaHTable(connection);
1201 try {
1202 addRegionToMeta(meta, regionInfo, splitA, splitB);
1203 } finally {
1204 meta.close();
1205 }
1206 }
1207
1208
1209
1210
1211
1212
1213
1214 public static void addRegionsToMeta(Connection connection,
1215 List<HRegionInfo> regionInfos, int regionReplication)
1216 throws IOException {
1217 addRegionsToMeta(connection, regionInfos, regionReplication, HConstants.LATEST_TIMESTAMP);
1218 }
1219
1220
1221
1222
1223
1224
1225
1226
1227 public static void addRegionsToMeta(Connection connection,
1228 List<HRegionInfo> regionInfos, int regionReplication, long ts)
1229 throws IOException {
1230 List<Put> puts = new ArrayList<Put>();
1231 for (HRegionInfo regionInfo : regionInfos) {
1232 if (RegionReplicaUtil.isDefaultReplica(regionInfo)) {
1233 Put put = makePutFromRegionInfo(regionInfo, ts);
1234
1235
1236 for (int i = 1; i < regionReplication; i++) {
1237 addEmptyLocation(put, i);
1238 }
1239 puts.add(put);
1240 }
1241 }
1242 putsToMetaTable(connection, puts);
1243 LOG.info("Added " + puts.size());
1244 }
1245
1246
1247
1248
1249
1250
1251
1252 public static void addDaughter(final Connection connection,
1253 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
1254 throws NotAllMetaRegionsOnlineException, IOException {
1255 Put put = new Put(regionInfo.getRegionName());
1256 addRegionInfo(put, regionInfo);
1257 if (sn != null) {
1258 addLocation(put, sn, openSeqNum, -1, regionInfo.getReplicaId());
1259 }
1260 putToMetaTable(connection, put);
1261 LOG.info("Added daughter " + regionInfo.getEncodedName() +
1262 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
1263 }
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277 public static void mergeRegions(final Connection connection, HRegionInfo mergedRegion,
1278 HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication,
1279 long masterSystemTime)
1280 throws IOException {
1281 Table meta = getMetaHTable(connection);
1282 try {
1283 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
1284
1285
1286 long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
1287
1288
1289 Put putOfMerged = makePutFromRegionInfo(copyOfMerged, time);
1290 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
1291 regionA.toByteArray());
1292 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
1293 regionB.toByteArray());
1294
1295
1296 Delete deleteA = makeDeleteFromRegionInfo(regionA, time);
1297 Delete deleteB = makeDeleteFromRegionInfo(regionB, time);
1298
1299
1300 addLocation(putOfMerged, sn, 1, -1, mergedRegion.getReplicaId());
1301
1302
1303
1304 for (int i = 1; i < regionReplication; i++) {
1305 addEmptyLocation(putOfMerged, i);
1306 }
1307
1308 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
1309 + HConstants.DELIMITER);
1310 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
1311 } finally {
1312 meta.close();
1313 }
1314 }
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327 public static void splitRegion(final Connection connection,
1328 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
1329 ServerName sn, int regionReplication) throws IOException {
1330 Table meta = getMetaHTable(connection);
1331 try {
1332 HRegionInfo copyOfParent = new HRegionInfo(parent);
1333 copyOfParent.setOffline(true);
1334 copyOfParent.setSplit(true);
1335
1336
1337 Put putParent = makePutFromRegionInfo(copyOfParent);
1338 addDaughtersToPut(putParent, splitA, splitB);
1339
1340
1341 Put putA = makePutFromRegionInfo(splitA);
1342 Put putB = makePutFromRegionInfo(splitB);
1343
1344 addSequenceNum(putA, 1, -1, splitA.getReplicaId());
1345 addSequenceNum(putB, 1, -1, splitB.getReplicaId());
1346
1347
1348
1349 for (int i = 1; i < regionReplication; i++) {
1350 addEmptyLocation(putA, i);
1351 addEmptyLocation(putB, i);
1352 }
1353
1354 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
1355 multiMutate(meta, tableRow, putParent, putA, putB);
1356 } finally {
1357 meta.close();
1358 }
1359 }
1360
1361
1362
1363
1364 private static void multiMutate(Table table, byte[] row, Mutation... mutations)
1365 throws IOException {
1366 CoprocessorRpcChannel channel = table.coprocessorService(row);
1367 MultiRowMutationProtos.MutateRowsRequest.Builder mmrBuilder
1368 = MultiRowMutationProtos.MutateRowsRequest.newBuilder();
1369 for (Mutation mutation : mutations) {
1370 if (mutation instanceof Put) {
1371 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(
1372 ClientProtos.MutationProto.MutationType.PUT, mutation));
1373 } else if (mutation instanceof Delete) {
1374 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(
1375 ClientProtos.MutationProto.MutationType.DELETE, mutation));
1376 } else {
1377 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
1378 + mutation.getClass().getName());
1379 }
1380 }
1381
1382 MultiRowMutationProtos.MultiRowMutationService.BlockingInterface service =
1383 MultiRowMutationProtos.MultiRowMutationService.newBlockingStub(channel);
1384 try {
1385 service.mutateRows(null, mmrBuilder.build());
1386 } catch (ServiceException ex) {
1387 ProtobufUtil.toIOException(ex);
1388 }
1389 }
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405 public static void updateRegionLocation(Connection connection,
1406 HRegionInfo regionInfo, ServerName sn, long openSeqNum,
1407 long masterSystemTime)
1408 throws IOException {
1409 updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime);
1410 }
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426 private static void updateLocation(final Connection connection,
1427 HRegionInfo regionInfo, ServerName sn, long openSeqNum,
1428 long masterSystemTime)
1429 throws IOException {
1430
1431
1432 long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
1433
1434
1435 Put put = new Put(getMetaKeyForRegion(regionInfo), time);
1436 HRegionInfo defaultRegionInfo = regionInfo;
1437 if (regionInfo.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
1438 defaultRegionInfo = new HRegionInfo(regionInfo, HRegionInfo.DEFAULT_REPLICA_ID);
1439 }
1440 addRegionInfo(put, defaultRegionInfo);
1441 addLocation(put, sn, openSeqNum, time, regionInfo.getReplicaId());
1442 putToMetaTable(connection, put);
1443 LOG.info("Updated row " + defaultRegionInfo.getRegionNameAsString() +
1444 " with server=" + sn);
1445 }
1446
1447
1448
1449
1450
1451
1452
1453 public static void deleteRegion(Connection connection,
1454 HRegionInfo regionInfo)
1455 throws IOException {
1456 Delete delete = new Delete(regionInfo.getRegionName());
1457 deleteFromMetaTable(connection, delete);
1458 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
1459 }
1460
1461
1462
1463
1464
1465
1466
1467
1468 public static void deleteRegions(Connection connection,
1469 List<HRegionInfo> regionsInfo, long ts) throws IOException {
1470 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
1471 for (HRegionInfo hri: regionsInfo) {
1472 deletes.add(new Delete(hri.getRegionName(), ts));
1473 }
1474 deleteFromMetaTable(connection, deletes);
1475 LOG.info("Deleted " + regionsInfo);
1476 }
1477
1478
1479
1480
1481
1482
1483 public static void deleteRegions(Connection connection,
1484 List<HRegionInfo> regionsInfo) throws IOException {
1485 deleteRegions(connection, regionsInfo, HConstants.LATEST_TIMESTAMP);
1486 }
1487
1488
1489
1490
1491
1492
1493
1494
1495 public static void mutateRegions(Connection connection,
1496 final List<HRegionInfo> regionsToRemove,
1497 final List<HRegionInfo> regionsToAdd)
1498 throws IOException {
1499 List<Mutation> mutation = new ArrayList<Mutation>();
1500 if (regionsToRemove != null) {
1501 for (HRegionInfo hri: regionsToRemove) {
1502 mutation.add(new Delete(hri.getRegionName()));
1503 }
1504 }
1505 if (regionsToAdd != null) {
1506 for (HRegionInfo hri: regionsToAdd) {
1507 mutation.add(makePutFromRegionInfo(hri));
1508 }
1509 }
1510 mutateMetaTable(connection, mutation);
1511 if (regionsToRemove != null && regionsToRemove.size() > 0) {
1512 LOG.debug("Deleted " + regionsToRemove);
1513 }
1514 if (regionsToAdd != null && regionsToAdd.size() > 0) {
1515 LOG.debug("Added " + regionsToAdd);
1516 }
1517 }
1518
1519
1520
1521
1522
1523
1524
1525 public static void overwriteRegions(Connection connection,
1526 List<HRegionInfo> regionInfos, int regionReplication) throws IOException {
1527
1528 long now = EnvironmentEdgeManager.currentTime();
1529 deleteRegions(connection, regionInfos, now);
1530
1531
1532
1533
1534
1535
1536 addRegionsToMeta(connection, regionInfos, regionReplication, now+1);
1537 LOG.info("Overwritten " + regionInfos);
1538 }
1539
1540
1541
1542
1543
1544
1545
1546 public static void deleteMergeQualifiers(Connection connection,
1547 final HRegionInfo mergedRegion) throws IOException {
1548 Delete delete = new Delete(mergedRegion.getRegionName());
1549 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
1550 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
1551 deleteFromMetaTable(connection, delete);
1552 LOG.info("Deleted references in merged region "
1553 + mergedRegion.getRegionNameAsString() + ", qualifier="
1554 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
1555 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
1556 }
1557
1558 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
1559 throws IOException {
1560 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
1561 hri.toByteArray());
1562 return p;
1563 }
1564
1565 public static Put addLocation(final Put p, final ServerName sn, long openSeqNum,
1566 long time, int replicaId){
1567 if (time <= 0) {
1568 time = EnvironmentEdgeManager.currentTime();
1569 }
1570 p.addImmutable(HConstants.CATALOG_FAMILY, getServerColumn(replicaId), time,
1571 Bytes.toBytes(sn.getHostAndPort()));
1572 p.addImmutable(HConstants.CATALOG_FAMILY, getStartCodeColumn(replicaId), time,
1573 Bytes.toBytes(sn.getStartcode()));
1574 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), time,
1575 Bytes.toBytes(openSeqNum));
1576 return p;
1577 }
1578
1579 public static Put addEmptyLocation(final Put p, int replicaId) {
1580 long now = EnvironmentEdgeManager.currentTime();
1581 p.addImmutable(HConstants.CATALOG_FAMILY, getServerColumn(replicaId), now, null);
1582 p.addImmutable(HConstants.CATALOG_FAMILY, getStartCodeColumn(replicaId), now, null);
1583 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), now, null);
1584 return p;
1585 }
1586
1587 public static Put addSequenceNum(final Put p, long openSeqNum, long time, int replicaId) {
1588 if (time <= 0) {
1589 time = EnvironmentEdgeManager.currentTime();
1590 }
1591 p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), time,
1592 Bytes.toBytes(openSeqNum));
1593 return p;
1594 }
1595
1596
1597
1598
1599
1600
1601
1602 public static boolean infoServerExists(Connection connection) throws IOException {
1603
1604 CollectingVisitor<Result> visitor = new CollectingVisitor<Result>() {
1605 @Override
1606 public boolean visit(Result r) throws IOException {
1607 if (r == null || r.isEmpty()) return true;
1608 RegionLocations locations = getRegionLocations(r);
1609 if (locations == null) return true;
1610 for (HRegionLocation loc : locations.getRegionLocations()) {
1611 if (loc != null) {
1612 if (loc.getServerName() != null) {
1613 add(r);
1614
1615 return false;
1616 }
1617 }
1618 }
1619 return true;
1620 }
1621
1622 @Override
1623 void add(Result r) {
1624 this.results.add(r);
1625 }
1626 };
1627 fullScan(connection, visitor);
1628
1629 return visitor.getResults().size() >= 1;
1630 }
1631 }