001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase.util; 019 020import java.io.ByteArrayOutputStream; 021import java.io.DataInput; 022import java.io.DataInputStream; 023import java.io.DataOutput; 024import java.io.IOException; 025import java.io.InputStream; 026import java.io.OutputStream; 027import java.math.BigDecimal; 028import java.math.BigInteger; 029import java.nio.ByteBuffer; 030import java.util.Arrays; 031import org.apache.hadoop.hbase.io.ByteBufferWriter; 032import org.apache.hadoop.hbase.io.util.StreamUtils; 033import org.apache.hadoop.hbase.nio.ByteBuff; 034import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent; 035import org.apache.hadoop.io.IOUtils; 036import org.apache.hadoop.io.WritableUtils; 037import org.apache.yetus.audience.InterfaceAudience; 038 039/** 040 * Utility functions for working with byte buffers, such as reading/writing variable-length long 041 * numbers. 042 * @deprecated This class will become IA.Private in HBase 3.0. Downstream folks shouldn't use it. 043 */ 044@Deprecated 045@InterfaceAudience.Public 046public final class ByteBufferUtils { 047 // "Compressed integer" serialization helper constants. 048 public final static int VALUE_MASK = 0x7f; 049 public final static int NEXT_BIT_SHIFT = 7; 050 public final static int NEXT_BIT_MASK = 1 << 7; 051 @InterfaceAudience.Private 052 final static boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable(); 053 public final static boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned(); 054 055 private ByteBufferUtils() { 056 } 057 058 static abstract class Comparer { 059 abstract int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2); 060 061 abstract int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2); 062 } 063 064 static abstract class Converter { 065 abstract short toShort(ByteBuffer buffer, int offset); 066 067 abstract int toInt(ByteBuffer buffer); 068 069 abstract int toInt(ByteBuffer buffer, int offset); 070 071 abstract long toLong(ByteBuffer buffer, int offset); 072 073 abstract void putInt(ByteBuffer buffer, int val); 074 075 abstract int putInt(ByteBuffer buffer, int index, int val); 076 077 abstract void putShort(ByteBuffer buffer, short val); 078 079 abstract int putShort(ByteBuffer buffer, int index, short val); 080 081 abstract void putLong(ByteBuffer buffer, long val); 082 083 abstract int putLong(ByteBuffer buffer, int index, long val); 084 } 085 086 static class ComparerHolder { 087 static final String UNSAFE_COMPARER_NAME = ComparerHolder.class.getName() + "$UnsafeComparer"; 088 089 static final Comparer BEST_COMPARER = getBestComparer(); 090 091 static Comparer getBestComparer() { 092 try { 093 Class<? extends Comparer> theClass = 094 Class.forName(UNSAFE_COMPARER_NAME).asSubclass(Comparer.class); 095 096 return theClass.getConstructor().newInstance(); 097 } catch (Throwable t) { // ensure we really catch *everything* 098 return PureJavaComparer.INSTANCE; 099 } 100 } 101 102 static final class PureJavaComparer extends Comparer { 103 static final PureJavaComparer INSTANCE = new PureJavaComparer(); 104 105 private PureJavaComparer() { 106 } 107 108 @Override 109 public int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 110 int end1 = o1 + l1; 111 int end2 = o2 + l2; 112 for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) { 113 int a = buf1[i] & 0xFF; 114 int b = buf2.get(j) & 0xFF; 115 if (a != b) { 116 return a - b; 117 } 118 } 119 return l1 - l2; 120 } 121 122 @Override 123 public int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 124 int end1 = o1 + l1; 125 int end2 = o2 + l2; 126 for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) { 127 int a = buf1.get(i) & 0xFF; 128 int b = buf2.get(j) & 0xFF; 129 if (a != b) { 130 return a - b; 131 } 132 } 133 return l1 - l2; 134 } 135 } 136 137 static final class UnsafeComparer extends Comparer { 138 139 public UnsafeComparer() { 140 } 141 142 static { 143 if (!UNSAFE_UNALIGNED) { 144 throw new Error(); 145 } 146 } 147 148 @Override 149 public int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 150 long offset2Adj; 151 Object refObj2 = null; 152 if (buf2.isDirect()) { 153 offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2); 154 } else { 155 offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; 156 refObj2 = buf2.array(); 157 } 158 return compareToUnsafe(buf1, o1 + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET, l1, refObj2, 159 offset2Adj, l2); 160 } 161 162 @Override 163 public int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 164 long offset1Adj, offset2Adj; 165 Object refObj1 = null, refObj2 = null; 166 if (buf1.isDirect()) { 167 offset1Adj = o1 + UnsafeAccess.directBufferAddress(buf1); 168 } else { 169 offset1Adj = o1 + buf1.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; 170 refObj1 = buf1.array(); 171 } 172 if (buf2.isDirect()) { 173 offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2); 174 } else { 175 offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; 176 refObj2 = buf2.array(); 177 } 178 return compareToUnsafe(refObj1, offset1Adj, l1, refObj2, offset2Adj, l2); 179 } 180 } 181 } 182 183 static class ConverterHolder { 184 static final String UNSAFE_CONVERTER_NAME = 185 ConverterHolder.class.getName() + "$UnsafeConverter"; 186 static final Converter BEST_CONVERTER = getBestConverter(); 187 188 static Converter getBestConverter() { 189 try { 190 Class<? extends Converter> theClass = 191 Class.forName(UNSAFE_CONVERTER_NAME).asSubclass(Converter.class); 192 193 // yes, UnsafeComparer does implement Comparer<byte[]> 194 return theClass.getConstructor().newInstance(); 195 } catch (Throwable t) { // ensure we really catch *everything* 196 return PureJavaConverter.INSTANCE; 197 } 198 } 199 200 static final class PureJavaConverter extends Converter { 201 static final PureJavaConverter INSTANCE = new PureJavaConverter(); 202 203 private PureJavaConverter() { 204 } 205 206 @Override 207 short toShort(ByteBuffer buffer, int offset) { 208 return buffer.getShort(offset); 209 } 210 211 @Override 212 int toInt(ByteBuffer buffer) { 213 return buffer.getInt(); 214 } 215 216 @Override 217 int toInt(ByteBuffer buffer, int offset) { 218 return buffer.getInt(offset); 219 } 220 221 @Override 222 long toLong(ByteBuffer buffer, int offset) { 223 return buffer.getLong(offset); 224 } 225 226 @Override 227 void putInt(ByteBuffer buffer, int val) { 228 buffer.putInt(val); 229 } 230 231 @Override 232 int putInt(ByteBuffer buffer, int index, int val) { 233 buffer.putInt(index, val); 234 return index + Bytes.SIZEOF_INT; 235 } 236 237 @Override 238 void putShort(ByteBuffer buffer, short val) { 239 buffer.putShort(val); 240 } 241 242 @Override 243 int putShort(ByteBuffer buffer, int index, short val) { 244 buffer.putShort(index, val); 245 return index + Bytes.SIZEOF_SHORT; 246 } 247 248 @Override 249 void putLong(ByteBuffer buffer, long val) { 250 buffer.putLong(val); 251 } 252 253 @Override 254 int putLong(ByteBuffer buffer, int index, long val) { 255 buffer.putLong(index, val); 256 return index + Bytes.SIZEOF_LONG; 257 } 258 } 259 260 static final class UnsafeConverter extends Converter { 261 262 public UnsafeConverter() { 263 } 264 265 static { 266 if (!UNSAFE_UNALIGNED) { 267 throw new Error(); 268 } 269 } 270 271 @Override 272 short toShort(ByteBuffer buffer, int offset) { 273 return UnsafeAccess.toShort(buffer, offset); 274 } 275 276 @Override 277 int toInt(ByteBuffer buffer) { 278 int i = UnsafeAccess.toInt(buffer, buffer.position()); 279 buffer.position(buffer.position() + Bytes.SIZEOF_INT); 280 return i; 281 } 282 283 @Override 284 int toInt(ByteBuffer buffer, int offset) { 285 return UnsafeAccess.toInt(buffer, offset); 286 } 287 288 @Override 289 long toLong(ByteBuffer buffer, int offset) { 290 return UnsafeAccess.toLong(buffer, offset); 291 } 292 293 @Override 294 void putInt(ByteBuffer buffer, int val) { 295 int newPos = UnsafeAccess.putInt(buffer, buffer.position(), val); 296 buffer.position(newPos); 297 } 298 299 @Override 300 int putInt(ByteBuffer buffer, int index, int val) { 301 return UnsafeAccess.putInt(buffer, index, val); 302 } 303 304 @Override 305 void putShort(ByteBuffer buffer, short val) { 306 int newPos = UnsafeAccess.putShort(buffer, buffer.position(), val); 307 buffer.position(newPos); 308 } 309 310 @Override 311 int putShort(ByteBuffer buffer, int index, short val) { 312 return UnsafeAccess.putShort(buffer, index, val); 313 } 314 315 @Override 316 void putLong(ByteBuffer buffer, long val) { 317 int newPos = UnsafeAccess.putLong(buffer, buffer.position(), val); 318 buffer.position(newPos); 319 } 320 321 @Override 322 int putLong(ByteBuffer buffer, int index, long val) { 323 return UnsafeAccess.putLong(buffer, index, val); 324 } 325 } 326 } 327 328 /** 329 * Similar to {@link WritableUtils#writeVLong(java.io.DataOutput, long)}, but writes to a 330 * {@link ByteBuffer}. 331 */ 332 public static void writeVLong(ByteBuffer out, long i) { 333 if (i >= -112 && i <= 127) { 334 out.put((byte) i); 335 return; 336 } 337 338 int len = -112; 339 if (i < 0) { 340 i ^= -1L; // take one's complement 341 len = -120; 342 } 343 344 long tmp = i; 345 while (tmp != 0) { 346 tmp = tmp >> 8; 347 len--; 348 } 349 350 out.put((byte) len); 351 352 len = (len < -120) ? -(len + 120) : -(len + 112); 353 354 for (int idx = len; idx != 0; idx--) { 355 int shiftbits = (idx - 1) * 8; 356 long mask = 0xFFL << shiftbits; 357 out.put((byte) ((i & mask) >> shiftbits)); 358 } 359 } 360 361 private interface ByteVisitor { 362 byte get(); 363 } 364 365 private static long readVLong(ByteVisitor visitor) { 366 byte firstByte = visitor.get(); 367 int len = WritableUtils.decodeVIntSize(firstByte); 368 if (len == 1) { 369 return firstByte; 370 } 371 long i = 0; 372 for (int idx = 0; idx < len - 1; idx++) { 373 byte b = visitor.get(); 374 i = i << 8; 375 i = i | (b & 0xFF); 376 } 377 return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); 378 } 379 380 /** 381 * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuffer}. 382 */ 383 public static long readVLong(ByteBuffer in) { 384 return readVLong(in::get); 385 } 386 387 /** 388 * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a 389 * {@link ByteBuff}. 390 */ 391 public static long readVLong(ByteBuff in) { 392 return readVLong(in::get); 393 } 394 395 /** 396 * Put in buffer integer using 7 bit encoding. For each written byte: 7 bits are used to store 397 * value 1 bit is used to indicate whether there is next bit. 398 * @param value Int to be compressed. 399 * @param out Where to put compressed data 400 * @return Number of bytes written. 401 * @throws IOException on stream error 402 */ 403 public static int putCompressedInt(OutputStream out, final int value) throws IOException { 404 int i = 0; 405 int tmpvalue = value; 406 do { 407 byte b = (byte) (tmpvalue & VALUE_MASK); 408 tmpvalue >>>= NEXT_BIT_SHIFT; 409 if (tmpvalue != 0) { 410 b |= (byte) NEXT_BIT_MASK; 411 } 412 out.write(b); 413 i++; 414 } while (tmpvalue != 0); 415 return i; 416 } 417 418 /** 419 * Put in output stream 32 bit integer (Big Endian byte order). 420 * @param out Where to put integer. 421 * @param value Value of integer. 422 * @throws IOException On stream error. 423 */ 424 public static void putInt(OutputStream out, final int value) throws IOException { 425 // We have writeInt in ByteBufferOutputStream so that it can directly write 426 // int to underlying 427 // ByteBuffer in one step. 428 if (out instanceof ByteBufferWriter) { 429 ((ByteBufferWriter) out).writeInt(value); 430 } else { 431 StreamUtils.writeInt(out, value); 432 } 433 } 434 435 public static byte toByte(ByteBuffer buffer, int offset) { 436 if (UNSAFE_AVAIL) { 437 return UnsafeAccess.toByte(buffer, offset); 438 } else { 439 return buffer.get(offset); 440 } 441 } 442 443 /** 444 * Copy the data to the output stream and update position in buffer. 445 * @param out the stream to write bytes to 446 * @param in the buffer to read bytes from 447 * @param length the number of bytes to copy 448 */ 449 public static void moveBufferToStream(OutputStream out, ByteBuffer in, int length) 450 throws IOException { 451 copyBufferToStream(out, in, in.position(), length); 452 skip(in, length); 453 } 454 455 /** 456 * Copy data from a buffer to an output stream. Does not update the position in the buffer. 457 * @param out the stream to write bytes to 458 * @param in the buffer to read bytes from 459 * @param offset the offset in the buffer (from the buffer's array offset) to start copying bytes 460 * from 461 * @param length the number of bytes to copy 462 */ 463 public static void copyBufferToStream(OutputStream out, ByteBuffer in, int offset, int length) 464 throws IOException { 465 if (out instanceof ByteBufferWriter) { 466 ((ByteBufferWriter) out).write(in, offset, length); 467 } else if (in.hasArray()) { 468 out.write(in.array(), in.arrayOffset() + offset, length); 469 } else { 470 for (int i = 0; i < length; ++i) { 471 out.write(toByte(in, offset + i)); 472 } 473 } 474 } 475 476 /** 477 * Copy data from a buffer to an output stream. Does not update the position in the buffer. 478 * @param out the output stream to write bytes to 479 * @param in the buffer to read bytes from 480 * @param offset the offset in the buffer (from the buffer's array offset) to start copying bytes 481 * from 482 * @param length the number of bytes to copy 483 */ 484 public static void copyBufferToStream(DataOutput out, ByteBuffer in, int offset, int length) 485 throws IOException { 486 if (out instanceof ByteBufferWriter) { 487 ((ByteBufferWriter) out).write(in, offset, length); 488 } else if (in.hasArray()) { 489 out.write(in.array(), in.arrayOffset() + offset, length); 490 } else { 491 for (int i = 0; i < length; ++i) { 492 out.write(toByte(in, offset + i)); 493 } 494 } 495 } 496 497 public static int putLong(OutputStream out, final long value, final int fitInBytes) 498 throws IOException { 499 long tmpValue = value; 500 for (int i = 0; i < fitInBytes; ++i) { 501 out.write((byte) (tmpValue & 0xff)); 502 tmpValue >>>= 8; 503 } 504 return fitInBytes; 505 } 506 507 public static int putByte(ByteBuffer buffer, int offset, byte b) { 508 if (UNSAFE_AVAIL) { 509 return UnsafeAccess.putByte(buffer, offset, b); 510 } else { 511 buffer.put(offset, b); 512 return offset + 1; 513 } 514 } 515 516 /** 517 * Check how many bytes are required to store value. 518 * @param value Value which size will be tested. 519 * @return How many bytes are required to store value. 520 */ 521 public static int longFitsIn(final long value) { 522 if (value < 0) { 523 return 8; 524 } 525 526 if (value < (1L << (4 * 8))) { 527 // no more than 4 bytes 528 if (value < (1L << (2 * 8))) { 529 if (value < (1L << (1 * 8))) { 530 return 1; 531 } 532 return 2; 533 } 534 if (value < (1L << (3 * 8))) { 535 return 3; 536 } 537 return 4; 538 } 539 // more than 4 bytes 540 if (value < (1L << (6 * 8))) { 541 if (value < (1L << (5 * 8))) { 542 return 5; 543 } 544 return 6; 545 } 546 if (value < (1L << (7 * 8))) { 547 return 7; 548 } 549 return 8; 550 } 551 552 /** 553 * Check how many bytes is required to store value. 554 * @param value Value which size will be tested. 555 * @return How many bytes are required to store value. 556 */ 557 public static int intFitsIn(final int value) { 558 if (value < 0) { 559 return 4; 560 } 561 562 if (value < (1 << (2 * 8))) { 563 if (value < (1 << (1 * 8))) { 564 return 1; 565 } 566 return 2; 567 } 568 if (value <= (1 << (3 * 8))) { 569 return 3; 570 } 571 return 4; 572 } 573 574 /** 575 * Read integer from stream coded in 7 bits and increment position. 576 * @return the integer that has been read 577 */ 578 public static int readCompressedInt(InputStream input) throws IOException { 579 int result = 0; 580 int i = 0; 581 byte b; 582 do { 583 b = (byte) input.read(); 584 result += (b & VALUE_MASK) << (NEXT_BIT_SHIFT * i); 585 i++; 586 if (i > Bytes.SIZEOF_INT + 1) { 587 throw new IllegalStateException( 588 "Corrupted compressed int (too long: " + (i + 1) + " bytes)"); 589 } 590 } while (0 != (b & NEXT_BIT_MASK)); 591 return result; 592 } 593 594 /** 595 * Read integer from buffer coded in 7 bits and increment position. 596 * @return Read integer. 597 */ 598 public static int readCompressedInt(ByteBuffer buffer) { 599 byte b = buffer.get(); 600 if ((b & NEXT_BIT_MASK) != 0) { 601 return (b & VALUE_MASK) + (readCompressedInt(buffer) << NEXT_BIT_SHIFT); 602 } 603 return b & VALUE_MASK; 604 } 605 606 /** 607 * Read long which was written to fitInBytes bytes and increment position. 608 * @param fitInBytes In how many bytes given long is stored. 609 * @return The value of parsed long. 610 */ 611 public static long readLong(InputStream in, final int fitInBytes) throws IOException { 612 long tmpLong = 0; 613 for (int i = 0; i < fitInBytes; ++i) { 614 tmpLong |= (in.read() & 0xffL) << (8 * i); 615 } 616 return tmpLong; 617 } 618 619 /** 620 * Read long which was written to fitInBytes bytes and increment position. 621 * @param fitInBytes In how many bytes given long is stored. 622 * @return The value of parsed long. 623 */ 624 public static long readLong(ByteBuffer in, final int fitInBytes) { 625 long tmpLength = 0; 626 for (int i = 0; i < fitInBytes; ++i) { 627 tmpLength |= (in.get() & 0xffL) << (8L * i); 628 } 629 return tmpLength; 630 } 631 632 /** 633 * Copy the given number of bytes from the given stream and put it at the current position of the 634 * given buffer, updating the position in the buffer. 635 * @param out the buffer to write data to 636 * @param in the stream to read data from 637 * @param length the number of bytes to read/write 638 */ 639 public static void copyFromStreamToBuffer(ByteBuffer out, DataInputStream in, int length) 640 throws IOException { 641 if (out.hasArray()) { 642 in.readFully(out.array(), out.position() + out.arrayOffset(), length); 643 skip(out, length); 644 } else { 645 for (int i = 0; i < length; ++i) { 646 out.put(in.readByte()); 647 } 648 } 649 } 650 651 /** 652 * Copy from the InputStream to a new heap ByteBuffer until the InputStream is exhausted. 653 */ 654 public static ByteBuffer drainInputStreamToBuffer(InputStream is) throws IOException { 655 ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); 656 IOUtils.copyBytes(is, baos, 4096, true); 657 ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); 658 buffer.rewind(); 659 return buffer; 660 } 661 662 /** 663 * Copy one buffer's whole data to another. Write starts at the current position of 'out' buffer. 664 * Note : This will advance the position marker of {@code out} and also change the position maker 665 * for {@code in}. 666 * @param in source buffer 667 * @param out destination buffer 668 */ 669 public static void copyFromBufferToBuffer(ByteBuffer in, ByteBuffer out) { 670 if (in.hasArray() && out.hasArray()) { 671 int length = in.remaining(); 672 System.arraycopy(in.array(), in.arrayOffset(), out.array(), out.arrayOffset(), length); 673 out.position(out.position() + length); 674 in.position(in.limit()); 675 } else if (UNSAFE_AVAIL) { 676 int length = in.remaining(); 677 UnsafeAccess.copy(in, in.position(), out, out.position(), length); 678 out.position(out.position() + length); 679 in.position(in.limit()); 680 } else { 681 out.put(in); 682 } 683 } 684 685 /** 686 * Copy from one buffer to another from given offset. This will be absolute positional copying and 687 * won't affect the position of any of the buffers. 688 */ 689 public static void copyFromBufferToBuffer(ByteBuffer in, ByteBuffer out, int sourceOffset, 690 int destinationOffset, int length) { 691 if (in.hasArray() && out.hasArray()) { 692 System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), 693 out.arrayOffset() + destinationOffset, length); 694 } else if (UNSAFE_AVAIL) { 695 UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length); 696 } else { 697 ByteBuffer outDup = out.duplicate(); 698 outDup.position(destinationOffset); 699 ByteBuffer inDup = in.duplicate(); 700 inDup.position(sourceOffset).limit(sourceOffset + length); 701 outDup.put(inDup); 702 } 703 // We used to return a result but disabled; return destinationOffset + length; 704 } 705 706 /** 707 * Copy from one buffer to another from given offset. 708 * <p> 709 * Note : This will advance the position marker of {@code out} but not change the position maker 710 * for {@code in} 711 * @param in source buffer 712 * @param out destination buffer 713 * @param sourceOffset offset in the source buffer 714 * @param length how many bytes to copy 715 */ 716 public static void copyFromBufferToBuffer(ByteBuffer in, ByteBuffer out, int sourceOffset, 717 int length) { 718 if (in.hasArray() && out.hasArray()) { 719 System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), 720 out.position() + out.arrayOffset(), length); 721 skip(out, length); 722 } else if (UNSAFE_AVAIL) { 723 UnsafeAccess.copy(in, sourceOffset, out, out.position(), length); 724 skip(out, length); 725 } else { 726 ByteBuffer inDup = in.duplicate(); 727 inDup.position(sourceOffset).limit(sourceOffset + length); 728 out.put(inDup); 729 } 730 } 731 732 /** 733 * Find length of common prefix of two parts in the buffer 734 * @param buffer Where parts are located. 735 * @param offsetLeft Offset of the first part. 736 * @param offsetRight Offset of the second part. 737 * @param limit Maximal length of common prefix. 738 * @return Length of prefix. 739 */ 740 @SuppressWarnings("unused") 741 public static int findCommonPrefix(ByteBuffer buffer, int offsetLeft, int offsetRight, 742 int limit) { 743 int prefix = 0; 744 745 for (; prefix < limit; ++prefix) { 746 if (buffer.get(offsetLeft + prefix) != buffer.get(offsetRight + prefix)) { 747 break; 748 } 749 } 750 751 return prefix; 752 } 753 754 /** 755 * Find length of common prefix in two arrays. 756 * @param left Array to be compared. 757 * @param leftOffset Offset in left array. 758 * @param leftLength Length of left array. 759 * @param right Array to be compared. 760 * @param rightOffset Offset in right array. 761 * @param rightLength Length of right array. 762 */ 763 public static int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right, 764 int rightOffset, int rightLength) { 765 int length = Math.min(leftLength, rightLength); 766 int result = 0; 767 768 while (result < length && left[leftOffset + result] == right[rightOffset + result]) { 769 result++; 770 } 771 772 return result; 773 } 774 775 /** 776 * Find length of common prefix in two arrays. 777 * @param left ByteBuffer to be compared. 778 * @param leftOffset Offset in left ByteBuffer. 779 * @param leftLength Length of left ByteBuffer. 780 * @param right ByteBuffer to be compared. 781 * @param rightOffset Offset in right ByteBuffer. 782 * @param rightLength Length of right ByteBuffer. 783 */ 784 public static int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, 785 ByteBuffer right, int rightOffset, int rightLength) { 786 int length = Math.min(leftLength, rightLength); 787 int result = 0; 788 789 while ( 790 result < length && ByteBufferUtils.toByte(left, leftOffset + result) 791 == ByteBufferUtils.toByte(right, rightOffset + result) 792 ) { 793 result++; 794 } 795 796 return result; 797 } 798 799 /** 800 * Check whether two parts in the same buffer are equal. 801 * @param buffer In which buffer there are parts 802 * @param offsetLeft Beginning of first part. 803 * @param lengthLeft Length of the first part. 804 * @param offsetRight Beginning of the second part. 805 * @param lengthRight Length of the second part. 806 * @return True if equal 807 */ 808 public static boolean arePartsEqual(ByteBuffer buffer, int offsetLeft, int lengthLeft, 809 int offsetRight, int lengthRight) { 810 if (lengthLeft != lengthRight) { 811 return false; 812 } 813 814 if (buffer.hasArray()) { 815 return 0 == Bytes.compareTo(buffer.array(), buffer.arrayOffset() + offsetLeft, lengthLeft, 816 buffer.array(), buffer.arrayOffset() + offsetRight, lengthRight); 817 } 818 819 for (int i = 0; i < lengthRight; ++i) { 820 if (buffer.get(offsetLeft + i) != buffer.get(offsetRight + i)) { 821 return false; 822 } 823 } 824 return true; 825 } 826 827 /** 828 * Increment position in buffer. 829 * @param buffer In this buffer. 830 * @param length By that many bytes. 831 */ 832 public static void skip(ByteBuffer buffer, int length) { 833 buffer.position(buffer.position() + length); 834 } 835 836 public static void extendLimit(ByteBuffer buffer, int numBytes) { 837 buffer.limit(buffer.limit() + numBytes); 838 } 839 840 /** 841 * Copy the bytes from position to limit into a new byte[] of the exact length and sets the 842 * position and limit back to their original values (though not thread safe). 843 * @param buffer copy from here 844 * @param startPosition put buffer.get(startPosition) into byte[0] 845 * @return a new byte[] containing the bytes in the specified range 846 */ 847 public static byte[] toBytes(ByteBuffer buffer, int startPosition) { 848 int originalPosition = buffer.position(); 849 byte[] output = new byte[buffer.limit() - startPosition]; 850 buffer.position(startPosition); 851 buffer.get(output); 852 buffer.position(originalPosition); 853 return output; 854 } 855 856 /** 857 * Copy the given number of bytes from specified offset into a new byte[] 858 * @return a new byte[] containing the bytes in the specified range 859 */ 860 public static byte[] toBytes(ByteBuffer buffer, int offset, int length) { 861 byte[] output = new byte[length]; 862 for (int i = 0; i < length; i++) { 863 output[i] = buffer.get(offset + i); 864 } 865 return output; 866 } 867 868 public static boolean equals(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 869 if ((l1 == 0) || (l2 == 0)) { 870 // both 0 length, return true, or else false 871 return l1 == l2; 872 } 873 // Since we're often comparing adjacent sorted data, 874 // it's usual to have equal arrays except for the very last byte 875 // so check that first 876 if (toByte(buf1, o1 + l1 - 1) != toByte(buf2, o2 + l2 - 1)) return false; 877 return compareTo(buf1, o1, l1, buf2, o2, l2) == 0; 878 } 879 880 /** 881 * ByteBuffer to hash offset to start from length to hash 882 */ 883 public static int hashCode(ByteBuffer buf, int offset, int length) { 884 int hash = 1; 885 for (int i = offset; i < offset + length; i++) { 886 hash = (31 * hash) + (int) toByte(buf, i); 887 } 888 return hash; 889 } 890 891 public static int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 892 return ComparerHolder.BEST_COMPARER.compareTo(buf1, o1, l1, buf2, o2, l2); 893 } 894 895 public static boolean equals(ByteBuffer buf1, int o1, int l1, byte[] buf2, int o2, int l2) { 896 if ((l1 == 0) || (l2 == 0)) { 897 // both 0 length, return true, or else false 898 return l1 == l2; 899 } 900 // Since we're often comparing adjacent sorted data, 901 // it's usual to have equal arrays except for the very last byte 902 // so check that first 903 if (toByte(buf1, o1 + l1 - 1) != buf2[o2 + l2 - 1]) return false; 904 return compareTo(buf1, o1, l1, buf2, o2, l2) == 0; 905 } 906 907 // The below two methods show up in lots of places. Versions of them in commons util and in 908 // Cassandra. In guava too? They are copied from ByteBufferUtils. They are here as static 909 // privates. Seems to make code smaller and make Hotspot happier (comes of compares and study 910 // of compiled code via jitwatch). 911 912 public static int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2) { 913 return ComparerHolder.BEST_COMPARER.compareTo(buf1, o1, l1, buf2, o2, l2); 914 } 915 916 public static int compareTo(ByteBuffer buf1, int o1, int l1, byte[] buf2, int o2, int l2) { 917 return compareTo(buf2, o2, l2, buf1, o1, l1) * -1; 918 } 919 920 static int compareToUnsafe(Object obj1, long o1, int l1, Object obj2, long o2, int l2) { 921 final int stride = 8; 922 final int minLength = Math.min(l1, l2); 923 int strideLimit = minLength & ~(stride - 1); 924 int i; 925 926 /* 927 * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a time is no slower than 928 * comparing 4 bytes at a time even on 32-bit. On the other hand, it is substantially faster on 929 * 64-bit. 930 */ 931 for (i = 0; i < strideLimit; i += stride) { 932 long lw = HBasePlatformDependent.getLong(obj1, o1 + (long) i); 933 long rw = HBasePlatformDependent.getLong(obj2, o2 + (long) i); 934 if (lw != rw) { 935 if (!UnsafeAccess.LITTLE_ENDIAN) { 936 return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1; 937 } 938 939 /* 940 * We want to compare only the first index where left[index] != right[index]. This 941 * corresponds to the least significant nonzero byte in lw ^ rw, since lw and rw are 942 * little-endian. Long.numberOfTrailingZeros(diff) tells us the least significant nonzero 943 * bit, and zeroing out the first three bits of L.nTZ gives us the shift to get that least 944 * significant nonzero byte. This comparison logic is based on UnsignedBytes from guava v21 945 */ 946 int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7; 947 return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF)); 948 } 949 } 950 951 // The epilogue to cover the last (minLength % stride) elements. 952 for (; i < minLength; i++) { 953 int il = (HBasePlatformDependent.getByte(obj1, o1 + i) & 0xFF); 954 int ir = (HBasePlatformDependent.getByte(obj2, o2 + i) & 0xFF); 955 if (il != ir) { 956 return il - ir; 957 } 958 } 959 return l1 - l2; 960 } 961 962 /** 963 * Reads a short value at the given buffer's offset. 964 * @return short value at offset 965 */ 966 public static short toShort(ByteBuffer buffer, int offset) { 967 return ConverterHolder.BEST_CONVERTER.toShort(buffer, offset); 968 } 969 970 /** 971 * Reads an int value at the given buffer's current position. Also advances the buffer's position 972 */ 973 public static int toInt(ByteBuffer buffer) { 974 return ConverterHolder.BEST_CONVERTER.toInt(buffer); 975 } 976 977 /** 978 * Reads an int value at the given buffer's offset. 979 * @return int value at offset 980 */ 981 public static int toInt(ByteBuffer buffer, int offset) { 982 return ConverterHolder.BEST_CONVERTER.toInt(buffer, offset); 983 } 984 985 /** 986 * Converts a ByteBuffer to an int value 987 * @param buf The ByteBuffer 988 * @param offset Offset to int value 989 * @param length Number of bytes used to store the int value. 990 * @return the int value if there's not enough bytes left in the buffer after the given offset 991 */ 992 public static int readAsInt(ByteBuffer buf, int offset, final int length) { 993 if (offset + length > buf.limit()) { 994 throw new IllegalArgumentException("offset (" + offset + ") + length (" + length 995 + ") exceed the" + " limit of the buffer: " + buf.limit()); 996 } 997 int n = 0; 998 for (int i = offset; i < (offset + length); i++) { 999 n <<= 8; 1000 n ^= toByte(buf, i) & 0xFF; 1001 } 1002 return n; 1003 } 1004 1005 /** 1006 * Reads a long value at the given buffer's offset. 1007 * @return long value at offset 1008 */ 1009 public static long toLong(ByteBuffer buffer, int offset) { 1010 return ConverterHolder.BEST_CONVERTER.toLong(buffer, offset); 1011 } 1012 1013 /** 1014 * Put an int value out to the given ByteBuffer's current position in big-endian format. This also 1015 * advances the position in buffer by int size. 1016 * @param buffer the ByteBuffer to write to 1017 * @param val int to write out 1018 */ 1019 public static void putInt(ByteBuffer buffer, int val) { 1020 ConverterHolder.BEST_CONVERTER.putInt(buffer, val); 1021 } 1022 1023 public static int putInt(ByteBuffer buffer, int index, int val) { 1024 return ConverterHolder.BEST_CONVERTER.putInt(buffer, index, val); 1025 } 1026 1027 /** 1028 * Reads a double value at the given buffer's offset. 1029 * @param offset offset where double is 1030 * @return double value at offset 1031 */ 1032 public static double toDouble(ByteBuffer buffer, int offset) { 1033 return Double.longBitsToDouble(toLong(buffer, offset)); 1034 } 1035 1036 /** 1037 * Reads a BigDecimal value at the given buffer's offset. 1038 * @return BigDecimal value at offset 1039 */ 1040 public static BigDecimal toBigDecimal(ByteBuffer buffer, int offset, int length) { 1041 if (buffer == null || length < Bytes.SIZEOF_INT + 1 || (offset + length > buffer.limit())) { 1042 return null; 1043 } 1044 1045 int scale = toInt(buffer, offset); 1046 byte[] tcBytes = new byte[length - Bytes.SIZEOF_INT]; 1047 copyFromBufferToArray(tcBytes, buffer, offset + Bytes.SIZEOF_INT, 0, length - Bytes.SIZEOF_INT); 1048 return new BigDecimal(new BigInteger(tcBytes), scale); 1049 } 1050 1051 /** 1052 * Put a short value out to the given ByteBuffer's current position in big-endian format. This 1053 * also advances the position in buffer by short size. 1054 * @param buffer the ByteBuffer to write to 1055 * @param val short to write out 1056 */ 1057 public static void putShort(ByteBuffer buffer, short val) { 1058 ConverterHolder.BEST_CONVERTER.putShort(buffer, val); 1059 } 1060 1061 public static int putShort(ByteBuffer buffer, int index, short val) { 1062 return ConverterHolder.BEST_CONVERTER.putShort(buffer, index, val); 1063 } 1064 1065 public static int putAsShort(ByteBuffer buf, int index, int val) { 1066 buf.put(index + 1, (byte) val); 1067 val >>= 8; 1068 buf.put(index, (byte) val); 1069 return index + Bytes.SIZEOF_SHORT; 1070 } 1071 1072 /** 1073 * Put a long value out to the given ByteBuffer's current position in big-endian format. This also 1074 * advances the position in buffer by long size. 1075 * @param buffer the ByteBuffer to write to 1076 * @param val long to write out 1077 */ 1078 public static void putLong(ByteBuffer buffer, long val) { 1079 ConverterHolder.BEST_CONVERTER.putLong(buffer, val); 1080 } 1081 1082 public static int putLong(ByteBuffer buffer, int index, long val) { 1083 return ConverterHolder.BEST_CONVERTER.putLong(buffer, index, val); 1084 } 1085 1086 /** 1087 * Copies the bytes from given array's offset to length part into the given buffer. Puts the bytes 1088 * to buffer's current position. This also advances the position in the 'out' buffer by 'length' 1089 */ 1090 public static void copyFromArrayToBuffer(ByteBuffer out, byte[] in, int inOffset, int length) { 1091 if (out.hasArray()) { 1092 System.arraycopy(in, inOffset, out.array(), out.arrayOffset() + out.position(), length); 1093 // Move the position in out by length 1094 out.position(out.position() + length); 1095 } else if (UNSAFE_AVAIL) { 1096 UnsafeAccess.copy(in, inOffset, out, out.position(), length); 1097 // Move the position in out by length 1098 out.position(out.position() + length); 1099 } else { 1100 out.put(in, inOffset, length); 1101 } 1102 } 1103 1104 /** 1105 * Copies bytes from given array's offset to length part into the given buffer. Puts the bytes to 1106 * buffer's given position. This doesn't affact the position of buffer. 1107 */ 1108 public static void copyFromArrayToBuffer(ByteBuffer out, int outOffset, byte[] in, int inOffset, 1109 int length) { 1110 if (out.hasArray()) { 1111 System.arraycopy(in, inOffset, out.array(), out.arrayOffset() + outOffset, length); 1112 } else if (UNSAFE_AVAIL) { 1113 UnsafeAccess.copy(in, inOffset, out, outOffset, length); 1114 } else { 1115 ByteBuffer outDup = out.duplicate(); 1116 outDup.position(outOffset); 1117 outDup.put(in, inOffset, length); 1118 } 1119 } 1120 1121 /** 1122 * Copies specified number of bytes from given offset of 'in' ByteBuffer to the array. This 1123 * doesn't affact the position of buffer. 1124 */ 1125 public static void copyFromBufferToArray(byte[] out, ByteBuffer in, int sourceOffset, 1126 int destinationOffset, int length) { 1127 if (in.hasArray()) { 1128 System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out, destinationOffset, length); 1129 } else if (UNSAFE_AVAIL) { 1130 UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length); 1131 } else { 1132 ByteBuffer inDup = in.duplicate(); 1133 inDup.position(sourceOffset); 1134 inDup.get(out, destinationOffset, length); 1135 } 1136 } 1137 1138 /** 1139 * Similar to {@link Arrays#copyOfRange(byte[], int, int)} 1140 * @param original the buffer from which the copy has to happen 1141 * @param from the starting index 1142 * @param to the ending index 1143 * @return a byte[] created out of the copy 1144 */ 1145 public static byte[] copyOfRange(ByteBuffer original, int from, int to) { 1146 int newLength = to - from; 1147 if (newLength < 0) throw new IllegalArgumentException(from + " > " + to); 1148 byte[] copy = new byte[newLength]; 1149 ByteBufferUtils.copyFromBufferToArray(copy, original, from, 0, newLength); 1150 return copy; 1151 } 1152 1153 // For testing purpose 1154 public static String toStringBinary(final ByteBuffer b, int off, int len) { 1155 StringBuilder result = new StringBuilder(); 1156 // Just in case we are passed a 'len' that is > buffer length... 1157 if (off >= b.capacity()) return result.toString(); 1158 if (off + len > b.capacity()) len = b.capacity() - off; 1159 for (int i = off; i < off + len; ++i) { 1160 int ch = b.get(i) & 0xFF; 1161 if ( 1162 (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') 1163 || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 1164 ) { 1165 result.append((char) ch); 1166 } else { 1167 result.append(String.format("\\x%02X", ch)); 1168 } 1169 } 1170 return result.toString(); 1171 } 1172 1173 public static String toStringBinary(final ByteBuffer b) { 1174 return toStringBinary(b, 0, b.capacity()); 1175 } 1176 1177 /** 1178 * Find index of passed delimiter. 1179 * @return Index of delimiter having started from start of <code>b</code> moving rightward. 1180 */ 1181 public static int searchDelimiterIndex(ByteBuffer b, int offset, final int length, 1182 final int delimiter) { 1183 for (int i = offset, n = offset + length; i < n; i++) { 1184 if (b.get(i) == delimiter) { 1185 return i; 1186 } 1187 } 1188 return -1; 1189 } 1190 1191 /** 1192 * Find index of passed delimiter walking from end of buffer backwards. 1193 * @return Index of delimiter 1194 */ 1195 public static int searchDelimiterIndexInReverse(ByteBuffer b, int offset, int length, 1196 int delimiter) { 1197 for (int i = offset + length - 1; i >= offset; i--) { 1198 if (b.get(i) == delimiter) { 1199 return i; 1200 } 1201 } 1202 return -1; 1203 } 1204}