1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import com.google.protobuf.ByteString;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.Iterator;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.NavigableMap;
31 import java.util.TreeMap;
32 import java.util.UUID;
33 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.classification.InterfaceAudience;
37 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
38 import org.apache.hadoop.hbase.protobuf.generated.WALProtos;
39 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FamilyScope;
40 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.ScopeType;
41 import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
42 import org.apache.hadoop.hbase.regionserver.SequenceId;
43 import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
44 import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
45 import org.apache.hadoop.hbase.util.ByteStringer;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
68 public class WALKey implements SequenceId, Comparable<WALKey> {
69
70 @InterfaceAudience.Private
71 public MultiVersionConcurrencyControl getMvcc() {
72 return mvcc;
73 }
74
75
76
77
78
79
80
81
82
83 @InterfaceAudience.Private
84 public MultiVersionConcurrencyControl.WriteEntry getWriteEntry() throws InterruptedIOException {
85 assert this.writeEntry != null;
86 return this.writeEntry;
87 }
88
89 @InterfaceAudience.Private
90 public void setWriteEntry(MultiVersionConcurrencyControl.WriteEntry writeEntry) {
91 assert this.writeEntry == null;
92 this.writeEntry = writeEntry;
93
94 this.logSeqNum = writeEntry.getWriteNumber();
95 }
96
97
98
99
100 @InterfaceAudience.Private
101 protected enum Version {
102 UNVERSIONED(0),
103
104 INITIAL(-1),
105
106
107 COMPRESSED(-2);
108
109 public final int code;
110 static final Version[] byCode;
111 static {
112 byCode = Version.values();
113 for (int i = 0; i < byCode.length; i++) {
114 if (byCode[i].code != -1 * i) {
115 throw new AssertionError("Values in this enum should be descending by one");
116 }
117 }
118 }
119
120 Version(int code) {
121 this.code = code;
122 }
123
124 public boolean atLeast(Version other) {
125 return code <= other.code;
126 }
127
128 public static Version fromCode(int code) {
129 return byCode[code * -1];
130 }
131 }
132
133
134
135
136
137 private static final String PREFIX_CLUSTER_KEY = ".";
138
139
140
141 @InterfaceAudience.Private
142 protected static final Version VERSION = Version.COMPRESSED;
143
144
145 public static final long NO_SEQUENCE_ID = -1;
146
147
148
149 @InterfaceAudience.Private
150 protected byte [] encodedRegionName;
151
152 @InterfaceAudience.Private
153 protected TableName tablename;
154
155 @InterfaceAudience.Private
156 protected long logSeqNum;
157 private long origLogSeqNum = 0;
158
159
160 @InterfaceAudience.Private
161 protected long writeTime;
162
163
164
165 @InterfaceAudience.Private
166 protected List<UUID> clusterIds;
167
168 private NavigableMap<byte[], Integer> scopes;
169
170 private long nonceGroup = HConstants.NO_NONCE;
171 private long nonce = HConstants.NO_NONCE;
172 private MultiVersionConcurrencyControl mvcc;
173 private MultiVersionConcurrencyControl.WriteEntry writeEntry;
174 private Map<String, byte[]> extendedAttributes;
175
176 public static final List<UUID> EMPTY_UUIDS = Collections.unmodifiableList(new ArrayList<UUID>());
177
178
179 @InterfaceAudience.Private
180 protected CompressionContext compressionContext;
181
182 public WALKey() {
183 init(null, null, 0L, HConstants.LATEST_TIMESTAMP,
184 new ArrayList<UUID>(), HConstants.NO_NONCE, HConstants.NO_NONCE, null, null);
185 }
186
187 @InterfaceAudience.Private
188 public WALKey(final byte[] encodedRegionName, final TableName tablename,
189 long logSeqNum,
190 final long now, UUID clusterId) {
191 List<UUID> clusterIds = new ArrayList<UUID>();
192 clusterIds.add(clusterId);
193 init(encodedRegionName, tablename, logSeqNum, now, clusterIds,
194 HConstants.NO_NONCE, HConstants.NO_NONCE, null, null);
195 }
196
197 public WALKey(final byte[] encodedRegionName, final TableName tablename) {
198 this(encodedRegionName, tablename, System.currentTimeMillis());
199 }
200
201 public WALKey(final byte[] encodedRegionName, final TableName tablename, final long now) {
202 this(encodedRegionName, tablename, now, null);
203 }
204
205 public WALKey(final byte[] encodedRegionName,
206 final TableName tablename,
207 final long now,
208 MultiVersionConcurrencyControl mvcc) {
209 this(encodedRegionName, tablename, now, mvcc, null);
210 }
211
212 public WALKey(final byte[] encodedRegionName,
213 final TableName tablename,
214 final long now,
215 MultiVersionConcurrencyControl mvcc,
216 Map<String, byte[]> extendedAttributes) {
217 init(encodedRegionName,
218 tablename,
219 NO_SEQUENCE_ID,
220 now,
221 EMPTY_UUIDS,
222 HConstants.NO_NONCE,
223 HConstants.NO_NONCE,
224 mvcc,
225 extendedAttributes);
226 }
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241 public WALKey(final byte[] encodedRegionName,
242 final TableName tablename,
243 long logSeqNum,
244 final long now,
245 List<UUID> clusterIds,
246 long nonceGroup,
247 long nonce,
248 MultiVersionConcurrencyControl mvcc) {
249 init(encodedRegionName, tablename, logSeqNum, now, clusterIds, nonceGroup, nonce, mvcc, null);
250 }
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266 public WALKey(final byte[] encodedRegionName, final TableName tablename,
267 final long now, List<UUID> clusterIds, long nonceGroup,
268 final long nonce, final MultiVersionConcurrencyControl mvcc) {
269 init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, nonceGroup, nonce, mvcc,
270 null);
271 }
272
273
274
275
276
277
278
279
280
281
282
283
284
285 public WALKey(final byte[] encodedRegionName,
286 final TableName tablename,
287 long logSeqNum,
288 long nonceGroup,
289 long nonce,
290 final MultiVersionConcurrencyControl mvcc) {
291 this(encodedRegionName, tablename, logSeqNum, nonceGroup, nonce, mvcc, null);
292 }
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307 public WALKey(final byte[] encodedRegionName,
308 final TableName tablename,
309 long logSeqNum,
310 long nonceGroup,
311 long nonce,
312 final MultiVersionConcurrencyControl mvcc,
313 Map<String, byte[]> extendedAttributes) {
314 init(encodedRegionName,
315 tablename,
316 logSeqNum,
317 EnvironmentEdgeManager.currentTime(),
318 EMPTY_UUIDS,
319 nonceGroup,
320 nonce,
321 mvcc,
322 extendedAttributes);
323 }
324
325 @InterfaceAudience.Private
326 protected void init(final byte[] encodedRegionName,
327 final TableName tablename,
328 long logSeqNum,
329 final long now,
330 List<UUID> clusterIds,
331 long nonceGroup,
332 long nonce,
333 MultiVersionConcurrencyControl mvcc,
334 Map<String, byte[]> extendedAttributes) {
335 this.logSeqNum = logSeqNum;
336 this.writeTime = now;
337 this.clusterIds = clusterIds;
338 this.encodedRegionName = encodedRegionName;
339 this.tablename = tablename;
340 this.nonceGroup = nonceGroup;
341 this.nonce = nonce;
342 this.mvcc = mvcc;
343 this.extendedAttributes = extendedAttributes;
344 }
345
346
347
348
349 public void setCompressionContext(CompressionContext compressionContext) {
350 this.compressionContext = compressionContext;
351 }
352
353
354 public byte [] getEncodedRegionName() {
355 return encodedRegionName;
356 }
357
358
359 public TableName getTablename() {
360 return tablename;
361 }
362
363
364 public long getLogSeqNum() {
365 return this.logSeqNum;
366 }
367
368
369
370
371
372 public void setOrigLogSeqNum(final long seqId) {
373 this.origLogSeqNum = seqId;
374 }
375
376
377
378
379
380 public long getOrigLogSeqNum() {
381 return this.origLogSeqNum;
382 }
383
384
385
386
387
388
389 @Override
390 public long getSequenceId() throws IOException {
391 return this.logSeqNum;
392 }
393
394
395
396
397 public long getWriteTime() {
398 return this.writeTime;
399 }
400
401 public NavigableMap<byte[], Integer> getScopes() {
402 return scopes;
403 }
404
405
406 public long getNonceGroup() {
407 return nonceGroup;
408 }
409
410
411 public long getNonce() {
412 return nonce;
413 }
414
415 public void setScopes(NavigableMap<byte[], Integer> scopes) {
416 this.scopes = scopes;
417 }
418
419 public void readOlderScopes(NavigableMap<byte[], Integer> scopes) {
420 if (scopes != null) {
421 Iterator<Map.Entry<byte[], Integer>> iterator = scopes.entrySet()
422 .iterator();
423 while (iterator.hasNext()) {
424 Map.Entry<byte[], Integer> scope = iterator.next();
425 String key = Bytes.toString(scope.getKey());
426 if (key.startsWith(PREFIX_CLUSTER_KEY)) {
427 addClusterId(UUID.fromString(key.substring(PREFIX_CLUSTER_KEY
428 .length())));
429 iterator.remove();
430 }
431 }
432 if (scopes.size() > 0) {
433 this.scopes = scopes;
434 }
435 }
436 }
437
438
439
440
441 public void addClusterId(UUID clusterId) {
442 if (!clusterIds.contains(clusterId)) {
443 clusterIds.add(clusterId);
444 }
445 }
446
447
448
449
450 public List<UUID> getClusterIds() {
451 return clusterIds;
452 }
453
454
455
456
457
458 public UUID getOriginatingClusterId(){
459 return clusterIds.isEmpty() ? HConstants.DEFAULT_CLUSTER_ID : clusterIds.get(0);
460 }
461
462
463
464
465
466
467 public void addExtendedAttribute(String attributeKey, byte[] attributeValue){
468 if (extendedAttributes == null){
469 extendedAttributes = new HashMap<String, byte[]>();
470 }
471 extendedAttributes.put(attributeKey, attributeValue);
472 }
473
474
475
476
477
478
479 public byte[] getExtendedAttribute(String attributeKey) {
480 return extendedAttributes != null ? extendedAttributes.get(attributeKey) : null;
481 }
482
483
484
485
486 public Map<String, byte[]> getExtendedAttributes() {
487 return extendedAttributes != null ? new HashMap<String, byte[]>(extendedAttributes) :
488 new HashMap<String, byte[]>();
489 }
490
491 @Override
492 public String toString() {
493 return tablename + "/" + Bytes.toString(encodedRegionName) + "/" +
494 logSeqNum;
495 }
496
497
498
499
500
501
502
503
504 public Map<String, Object> toStringMap() {
505 Map<String, Object> stringMap = new HashMap<String, Object>();
506 stringMap.put("table", tablename);
507 stringMap.put("region", Bytes.toStringBinary(encodedRegionName));
508 stringMap.put("sequence", logSeqNum);
509 Map<String, byte[]> extendedAttributes = getExtendedAttributes();
510 if (extendedAttributes != null){
511 for (Map.Entry<String, byte[]> entry : extendedAttributes.entrySet()){
512 stringMap.put(entry.getKey(), Bytes.toStringBinary(entry.getValue()));
513 }
514 }
515 return stringMap;
516 }
517
518 @Override
519 public boolean equals(Object obj) {
520 if (this == obj) {
521 return true;
522 }
523 if (obj == null || getClass() != obj.getClass()) {
524 return false;
525 }
526 return compareTo((WALKey)obj) == 0;
527 }
528
529 @Override
530 public int hashCode() {
531 int result = Bytes.hashCode(this.encodedRegionName);
532 result = (int) (result ^ this.logSeqNum);
533 result = (int) (result ^ this.writeTime);
534 return result;
535 }
536
537 @Override
538 public int compareTo(WALKey o) {
539 int result = Bytes.compareTo(this.encodedRegionName, o.encodedRegionName);
540 if (result == 0) {
541 if (this.logSeqNum < o.logSeqNum) {
542 result = -1;
543 } else if (this.logSeqNum > o.logSeqNum) {
544 result = 1;
545 }
546 if (result == 0) {
547 if (this.writeTime < o.writeTime) {
548 result = -1;
549 } else if (this.writeTime > o.writeTime) {
550 return 1;
551 }
552 }
553 }
554
555 return result;
556 }
557
558
559
560
561
562
563
564 void internTableName(TableName tablename) {
565
566
567 assert tablename.equals(this.tablename);
568 this.tablename = tablename;
569 }
570
571
572
573
574
575
576
577 void internEncodedRegionName(byte []encodedRegionName) {
578
579
580 assert Bytes.equals(this.encodedRegionName, encodedRegionName);
581 this.encodedRegionName = encodedRegionName;
582 }
583
584 public org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.Builder getBuilder(
585 WALCellCodec.ByteStringCompressor compressor) throws IOException {
586 org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.Builder builder =
587 org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.newBuilder();
588 if (compressionContext == null) {
589 builder.setEncodedRegionName(ByteStringer.wrap(this.encodedRegionName));
590 builder.setTableName(ByteStringer.wrap(this.tablename.getName()));
591 } else {
592 builder.setEncodedRegionName(compressor.compress(this.encodedRegionName,
593 compressionContext.regionDict));
594 builder.setTableName(compressor.compress(this.tablename.getName(),
595 compressionContext.tableDict));
596 }
597 builder.setLogSequenceNumber(this.logSeqNum);
598 builder.setWriteTime(writeTime);
599 if (this.origLogSeqNum > 0) {
600 builder.setOrigSequenceNumber(this.origLogSeqNum);
601 }
602 if (this.nonce != HConstants.NO_NONCE) {
603 builder.setNonce(nonce);
604 }
605 if (this.nonceGroup != HConstants.NO_NONCE) {
606 builder.setNonceGroup(nonceGroup);
607 }
608 HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder();
609 for (UUID clusterId : clusterIds) {
610 uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
611 uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
612 builder.addClusterIds(uuidBuilder.build());
613 }
614 if (scopes != null) {
615 for (Map.Entry<byte[], Integer> e : scopes.entrySet()) {
616 ByteString family = (compressionContext == null) ? ByteStringer.wrap(e.getKey())
617 : compressor.compress(e.getKey(), compressionContext.familyDict);
618 builder.addScopes(FamilyScope.newBuilder()
619 .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue())));
620 }
621 }
622 if (extendedAttributes != null){
623 for (Map.Entry<String, byte[]> e : extendedAttributes.entrySet()){
624 WALProtos.Attribute attr = WALProtos.Attribute.newBuilder().
625 setKey(e.getKey()).
626 setValue((compressionContext == null) ? ByteStringer.wrap(e.getValue())
627 : compressor.compress(e.getValue(), compressionContext.tableDict)).build();
628 builder.addExtendedAttributes(attr);
629 }
630 }
631 return builder;
632 }
633
634 public void readFieldsFromPb(org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey walKey,
635 WALCellCodec.ByteStringUncompressor uncompressor)
636 throws IOException {
637 if (this.compressionContext != null) {
638 this.encodedRegionName = uncompressor.uncompress(
639 walKey.getEncodedRegionName(), compressionContext.regionDict);
640 byte[] tablenameBytes = uncompressor.uncompress(
641 walKey.getTableName(), compressionContext.tableDict);
642 this.tablename = TableName.valueOf(tablenameBytes);
643 } else {
644 this.encodedRegionName = walKey.getEncodedRegionName().toByteArray();
645 this.tablename = TableName.valueOf(walKey.getTableName().toByteArray());
646 }
647 clusterIds.clear();
648 if (walKey.hasClusterId()) {
649
650
651 clusterIds.add(new UUID(walKey.getClusterId().getMostSigBits(), walKey.getClusterId()
652 .getLeastSigBits()));
653 }
654 for (HBaseProtos.UUID clusterId : walKey.getClusterIdsList()) {
655 clusterIds.add(new UUID(clusterId.getMostSigBits(), clusterId.getLeastSigBits()));
656 }
657 if (walKey.hasNonceGroup()) {
658 this.nonceGroup = walKey.getNonceGroup();
659 }
660 if (walKey.hasNonce()) {
661 this.nonce = walKey.getNonce();
662 }
663 this.scopes = null;
664 if (walKey.getScopesCount() > 0) {
665 this.scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
666 for (FamilyScope scope : walKey.getScopesList()) {
667 byte[] family = (compressionContext == null) ? scope.getFamily().toByteArray() :
668 uncompressor.uncompress(scope.getFamily(), compressionContext.familyDict);
669 this.scopes.put(family, scope.getScopeType().getNumber());
670 }
671 }
672 this.logSeqNum = walKey.getLogSequenceNumber();
673 this.writeTime = walKey.getWriteTime();
674 if (walKey.hasOrigSequenceNumber()) {
675 this.origLogSeqNum = walKey.getOrigSequenceNumber();
676 }
677 if (walKey.getExtendedAttributesCount() > 0){
678 this.extendedAttributes = new HashMap<>(walKey.getExtendedAttributesCount());
679 for (WALProtos.Attribute attr : walKey.getExtendedAttributesList()) {
680 byte[] value = (compressionContext == null) ? attr.getValue().toByteArray() :
681 uncompressor.uncompress(attr.getValue(), compressionContext.tableDict);
682 extendedAttributes.put(attr.getKey(), value);
683 }
684 }
685 }
686
687 public long estimatedSerializedSizeOf() {
688 long size = encodedRegionName != null ? encodedRegionName.length : 0;
689 size += tablename != null ? tablename.toBytes().length : 0;
690 if (clusterIds != null) {
691 size += 16 * clusterIds.size();
692 }
693 if (nonceGroup != HConstants.NO_NONCE) {
694 size += Bytes.SIZEOF_LONG;
695 }
696 if (nonce != HConstants.NO_NONCE) {
697 size += Bytes.SIZEOF_LONG;
698 }
699 if (scopes != null) {
700 for (Map.Entry<byte[], Integer> scope: scopes.entrySet()) {
701 size += scope.getKey().length;
702 size += Bytes.SIZEOF_INT;
703 }
704 }
705 size += Bytes.SIZEOF_LONG;
706 size += Bytes.SIZEOF_LONG;
707 if (origLogSeqNum > 0) {
708 size += Bytes.SIZEOF_LONG;
709 }
710 return size;
711 }
712 }