View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertNotEquals;
23  
24  import java.io.ByteArrayInputStream;
25  import java.io.ByteArrayOutputStream;
26  import java.io.DataInputStream;
27  import java.io.DataOutputStream;
28  import java.util.Collections;
29  import java.util.Iterator;
30  import java.util.List;
31  import java.util.Set;
32  import java.util.TreeSet;
33  
34  import junit.framework.TestCase;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.hbase.KeyValue.KVComparator;
39  import org.apache.hadoop.hbase.KeyValue.MetaComparator;
40  import org.apache.hadoop.hbase.KeyValue.Type;
41  import org.apache.hadoop.hbase.testclassification.SmallTests;
42  import org.apache.hadoop.hbase.util.ByteBufferUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  @Category(SmallTests.class)
48  public class TestKeyValue extends TestCase {
49    private static final Log LOG = LogFactory.getLog(TestKeyValue.class);
50  
51    public void testColumnCompare() {
52      final byte[] a = Bytes.toBytes("aaa");
53      byte[] family1 = Bytes.toBytes("abc");
54      byte[] qualifier1 = Bytes.toBytes("def");
55      byte[] family2 = Bytes.toBytes("abcd");
56      byte[] qualifier2 = Bytes.toBytes("ef");
57  
58      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
59      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
60      assertTrue(CellUtil.matchingColumn(aaa, family1, qualifier1));
61      aaa = new KeyValue(a, family2, qualifier2, 0L, KeyValue.Type.Put, a);
62      assertFalse(CellUtil.matchingColumn(aaa, family1, qualifier1));
63      assertTrue(CellUtil.matchingColumn(aaa, family2,qualifier2));
64      byte[] nullQualifier = new byte[0];
65      aaa = new KeyValue(a, family1, nullQualifier, 0L, KeyValue.Type.Put, a);
66      assertTrue(CellUtil.matchingColumn(aaa, family1,null));
67      assertFalse(CellUtil.matchingColumn(aaa, family2,qualifier2));
68    }
69  
70    /**
71     * Test a corner case when the family qualifier is a prefix of the column qualifier.
72     */
73    public void testColumnCompare_prefix() {
74      final byte[] a = Bytes.toBytes("aaa");
75      byte[] family1 = Bytes.toBytes("abc");
76      byte[] qualifier1 = Bytes.toBytes("def");
77      byte[] family2 = Bytes.toBytes("ab");
78      byte[] qualifier2 = Bytes.toBytes("def");
79  
80      KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, KeyValue.Type.Put, a);
81      assertFalse(CellUtil.matchingColumn(aaa, family2, qualifier2));
82    }
83  
84    public void testBasics() {
85      LOG.info("LOWKEY: " + KeyValue.LOWESTKEY.toString());
86      check(Bytes.toBytes(getName()),
87        Bytes.toBytes(getName()), Bytes.toBytes(getName()), 1,
88        Bytes.toBytes(getName()));
89      // Test empty value and empty column -- both should work. (not empty fam)
90      check(Bytes.toBytes(getName()), Bytes.toBytes(getName()), null, 1, null);
91      check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(getName()), null, 1, null);
92      // empty qual is equivalent to null qual
93      assertEquals(
94        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"), null, 1, (byte[]) null),
95        new KeyValue(Bytes.toBytes("rk"), Bytes.toBytes("fam"),
96          HConstants.EMPTY_BYTE_ARRAY, 1, (byte[]) null));
97    }
98  
99    private void check(final byte [] row, final byte [] family, byte [] qualifier,
100     final long timestamp, final byte [] value) {
101     KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value);
102     assertTrue(Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
103       row.length) == 0);
104     assertTrue(CellUtil.matchingColumn(kv, family, qualifier));
105     // Call toString to make sure it works.
106     LOG.info(kv.toString());
107   }
108 
109   public void testPlainCompare() {
110     final byte[] a = Bytes.toBytes("aaa");
111     final byte[] b = Bytes.toBytes("bbb");
112     final byte[] fam = Bytes.toBytes("col");
113     final byte[] qf = Bytes.toBytes("umn");
114     KeyValue aaa = new KeyValue(a, fam, qf, a);
115     KeyValue bbb = new KeyValue(b, fam, qf, b);
116     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
117     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
118     // Compare breaks if passed same ByteBuffer as both left and right arguments.
119     assertTrue(KeyValue.COMPARATOR.compare(bbb, bbb) == 0);
120     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
121     // Do compare with different timestamps.
122     aaa = new KeyValue(a, fam, qf, 1, a);
123     bbb = new KeyValue(a, fam, qf, 2, a);
124     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) > 0);
125     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) < 0);
126     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
127     // Do compare with different types.  Higher numbered types -- Delete
128     // should sort ahead of lower numbers; i.e. Put
129     aaa = new KeyValue(a, fam, qf, 1, KeyValue.Type.Delete, a);
130     bbb = new KeyValue(a, fam, qf, 1, a);
131     assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0);
132     assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0);
133     assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0);
134   }
135 
136   public void testMoreComparisons() {
137     long now = System.currentTimeMillis();
138 
139     // Meta compares
140     KeyValue aaa = new KeyValue(
141         Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now);
142     KeyValue bbb = new KeyValue(
143         Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now);
144     KVComparator c = new KeyValue.MetaComparator();
145     assertTrue(c.compare(bbb, aaa) < 0);
146 
147     KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"),
148         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L,
149         (byte[])null);
150     assertTrue(c.compare(aaaa, bbb) < 0);
151 
152     KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
153         Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L,
154         (byte[])null);
155     KeyValue y = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"),
156         Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236034574912L,
157         (byte[])null);
158     assertTrue(c.compare(x, y) < 0);
159     comparisons(new KeyValue.MetaComparator());
160     comparisons(new KeyValue.KVComparator());
161     metacomparisons(new KeyValue.MetaComparator());
162   }
163 
164   public void testMetaComparatorTableKeysWithCommaOk() {
165     MetaComparator c = new KeyValue.MetaComparator();
166     long now = System.currentTimeMillis();
167     // meta keys values are not quite right.  A users can enter illegal values
168     // from shell when scanning meta.
169     KeyValue a = new KeyValue(Bytes.toBytes("table,key,with,commas1,1234"), now);
170     KeyValue b = new KeyValue(Bytes.toBytes("table,key,with,commas2,0123"), now);
171     assertTrue(c.compare(a, b) < 0);
172   }
173 
174   /**
175    * Tests cases where rows keys have characters below the ','.
176    * See HBASE-832
177    */
178   public void testKeyValueBorderCases() {
179     // % sorts before , so if we don't do special comparator, rowB would
180     // come before rowA.
181     KeyValue rowA = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/,1234"),
182       Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
183     KeyValue rowB = new KeyValue(Bytes.toBytes("testtable,www.hbase.org/%20,99999"),
184         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
185     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
186 
187     rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"),
188         Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
189     rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"),
190         Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null);
191     assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0);
192   }
193 
194   private void metacomparisons(final KeyValue.MetaComparator c) {
195     long now = System.currentTimeMillis();
196     assertTrue(c.compare(new KeyValue(
197         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now),
198       new KeyValue(
199           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) == 0);
200     KeyValue a = new KeyValue(
201         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now);
202     KeyValue b = new KeyValue(
203         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now);
204     assertTrue(c.compare(a, b) < 0);
205     assertTrue(c.compare(new KeyValue(
206         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,2"), now),
207       new KeyValue(
208           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",a,,0,1"), now)) > 0);
209   }
210 
211   private void comparisons(final KeyValue.KVComparator c) {
212     long now = System.currentTimeMillis();
213     assertTrue(c.compare(new KeyValue(
214         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
215       new KeyValue(
216           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) == 0);
217     assertTrue(c.compare(new KeyValue(
218         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now),
219       new KeyValue(
220           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now)) < 0);
221     assertTrue(c.compare(new KeyValue(
222         Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,2"), now),
223       new KeyValue(
224           Bytes.toBytes(TableName.META_TABLE_NAME.getNameAsString()+",,1"), now)) > 0);
225   }
226 
227   public void testBinaryKeys() {
228     Set<KeyValue> set = new TreeSet<>(KeyValue.COMPARATOR);
229     final byte[] fam = Bytes.toBytes("col");
230     final byte[] qf = Bytes.toBytes("umn");
231     final byte[] nb = new byte[0];
232     KeyValue[] keys = {new KeyValue(Bytes.toBytes("aaaaa,\u0000\u0000,2"), fam, qf, 2, nb),
233       new KeyValue(Bytes.toBytes("aaaaa,\u0001,3"), fam, qf, 3, nb),
234       new KeyValue(Bytes.toBytes("aaaaa,,1"), fam, qf, 1, nb),
235       new KeyValue(Bytes.toBytes("aaaaa,\u1000,5"), fam, qf, 5, nb),
236       new KeyValue(Bytes.toBytes("aaaaa,a,4"), fam, qf, 4, nb),
237       new KeyValue(Bytes.toBytes("a,a,0"), fam, qf, 0, nb),
238     };
239     // Add to set with bad comparator
240     Collections.addAll(set, keys);
241     // This will output the keys incorrectly.
242     boolean assertion = false;
243     int count = 0;
244     try {
245       for (KeyValue k : set) {
246         assertEquals(count++, k.getTimestamp());
247       }
248     } catch (junit.framework.AssertionFailedError e) {
249       // Expected
250       assertion = true;
251     }
252     assertTrue(assertion);
253     // Make set with good comparator
254     set = new TreeSet<>(new KeyValue.MetaComparator());
255     Collections.addAll(set, keys);
256     count = 0;
257     for (KeyValue k : set) {
258       assertEquals(count++, k.getTimestamp());
259     }
260   }
261 
262   private final byte[] rowA = Bytes.toBytes("rowA");
263   private final byte[] rowB = Bytes.toBytes("rowB");
264 
265   private final byte[] family = Bytes.toBytes("family");
266   private final byte[] qualA = Bytes.toBytes("qfA");
267 
268   private void assertKVLess(KeyValue.KVComparator c, KeyValue less, KeyValue greater) {
269     int cmp = c.compare(less,greater);
270     assertTrue(cmp < 0);
271     cmp = c.compare(greater,less);
272     assertTrue(cmp > 0);
273   }
274 
275   private void assertKVLessWithoutRow(KeyValue.KVComparator c, int common, KeyValue less,
276       KeyValue greater) {
277     int cmp = c.compare(less, greater);
278     assertTrue(cmp < 0);
279     cmp = c.compare(greater, less);
280     assertTrue(cmp > 0);
281   }
282 
283   public void testCompareWithoutRow() {
284     final KVComparator c = new KeyValue.KVComparator();
285     byte[] row = Bytes.toBytes("row");
286 
287     byte[] fa = Bytes.toBytes("fa");
288     byte[] fami = Bytes.toBytes("fami");
289     byte[] fami1 = Bytes.toBytes("fami1");
290 
291     byte[] qual0 = Bytes.toBytes("");
292     byte[] qual1 = Bytes.toBytes("qf1");
293     byte[] qual2 = Bytes.toBytes("qf2");
294     long ts = 1;
295 
296     // 'fa:'
297     KeyValue kv_0 = new KeyValue(row, fa, qual0, ts, KeyValue.Type.Put);
298     // 'fami:'
299     KeyValue kv0_0 = new KeyValue(row, fami, qual0, ts, KeyValue.Type.Put);
300     // 'fami:qf1'
301     KeyValue kv0_1 = new KeyValue(row, fami, qual1, ts, KeyValue.Type.Put);
302     // 'fami:qf2'
303     KeyValue kv0_2 = new KeyValue(row, fami, qual2, ts, KeyValue.Type.Put);
304     // 'fami1:'
305     KeyValue kv1_0 = new KeyValue(row, fami1, qual0, ts, KeyValue.Type.Put);
306 
307     // 'fami:qf1' < 'fami:qf2'
308     assertKVLessWithoutRow(c, 0, kv0_1, kv0_2);
309     // 'fami:qf1' < 'fami1:'
310     assertKVLessWithoutRow(c, 0, kv0_1, kv1_0);
311 
312     // Test comparison by skipping the same prefix bytes.
313     /*
314      * KeyValue Format and commonLength:
315      * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|....
316      * ------------------|-------commonLength--------|--------------
317      */
318     int commonLength = KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE
319         + row.length;
320     // 'fa:' < 'fami:'. They have commonPrefix + 2 same prefix bytes.
321     assertKVLessWithoutRow(c, commonLength + 2, kv_0, kv0_0);
322     // 'fami:' < 'fami:qf1'. They have commonPrefix + 4 same prefix bytes.
323     assertKVLessWithoutRow(c, commonLength + 4, kv0_0, kv0_1);
324     // 'fami:qf1' < 'fami1:'. They have commonPrefix + 4 same prefix bytes.
325     assertKVLessWithoutRow(c, commonLength + 4, kv0_1, kv1_0);
326     // 'fami:qf1' < 'fami:qf2'. They have commonPrefix + 6 same prefix bytes.
327     assertKVLessWithoutRow(c, commonLength + 6, kv0_1, kv0_2);
328   }
329 
330   public void testFirstLastOnRow() {
331     final KVComparator c = new KeyValue.KVComparator();
332     long ts = 1;
333     byte[] bufferA = new byte[128];
334     int offsetA = 0;
335     byte[] bufferB = new byte[128];
336     int offsetB = 7;
337 
338     // These are listed in sort order (ie: every one should be less
339     // than the one on the next line).
340     final KeyValue firstOnRowA = KeyValueUtil.createFirstOnRow(rowA);
341     final KeyValue firstOnRowABufferFamQual = KeyValueUtil.createFirstOnRow(bufferA, offsetA,
342         rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
343     final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, KeyValue.Type.Put);
344     final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, KeyValue.Type.Put);
345 
346     final KeyValue lastOnRowA = KeyValueUtil.createLastOnRow(rowA);
347     final KeyValue firstOnRowB = KeyValueUtil.createFirstOnRow(rowB);
348     final KeyValue firstOnRowBBufferFam = KeyValueUtil.createFirstOnRow(bufferB, offsetB,
349         rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
350     final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, KeyValue.Type.Put);
351 
352     assertKVLess(c, firstOnRowA, firstOnRowB);
353     assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
354     assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
355     assertKVLess(c, firstOnRowA, kvA_1);
356     assertKVLess(c, firstOnRowA, kvA_2);
357     assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
358     assertKVLess(c, kvA_1, kvA_2);
359     assertKVLess(c, kvA_2, firstOnRowB);
360     assertKVLess(c, kvA_1, firstOnRowB);
361     assertKVLess(c, kvA_2, firstOnRowBBufferFam);
362     assertKVLess(c, kvA_1, firstOnRowBBufferFam);
363 
364     assertKVLess(c, lastOnRowA, firstOnRowB);
365     assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
366     assertKVLess(c, firstOnRowB, kvB);
367     assertKVLess(c, firstOnRowBBufferFam, kvB);
368     assertKVLess(c, lastOnRowA, kvB);
369 
370     assertKVLess(c, kvA_2, lastOnRowA);
371     assertKVLess(c, kvA_1, lastOnRowA);
372     assertKVLess(c, firstOnRowA, lastOnRowA);
373     assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
374   }
375 
376   public void testCreateKeyOnly() {
377     long ts = 1;
378     byte[] value = Bytes.toBytes("a real value");
379     byte[] evalue = new byte[0]; // empty value
380 
381     for (byte[] val : new byte[][] { value, evalue }) {
382       for (boolean useLen : new boolean[]{false,true}) {
383         KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
384         KeyValue kv1ko = kv1.createKeyOnly(useLen);
385         // keys are still the same
386         assertTrue(kv1.equals(kv1ko));
387         // but values are not
388         assertTrue(kv1ko.getValueLength() == (useLen?Bytes.SIZEOF_INT:0));
389         if (useLen) {
390           assertEquals(kv1.getValueLength(),
391             Bytes.toInt(kv1ko.getValueArray(), kv1ko.getValueOffset(), kv1ko.getValueLength()));
392         }
393       }
394     }
395   }
396 
397   public void testCreateKeyValueFromKey() {
398     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
399         Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("myValue"));
400     int initialPadding = 10;
401     int endingPadding = 20;
402     int keyLen = kv.getKeyLength();
403     byte[] tmpArr = new byte[initialPadding + endingPadding + keyLen];
404     System.arraycopy(kv.getBuffer(), kv.getKeyOffset(), tmpArr,
405         initialPadding, keyLen);
406     KeyValue kvFromKey = KeyValueUtil.createKeyValueFromKey(tmpArr, initialPadding,
407         keyLen);
408     assertEquals(keyLen, kvFromKey.getKeyLength());
409     assertEquals(KeyValue.ROW_OFFSET + keyLen, kvFromKey.getBuffer().length);
410     System.err.println("kv=" + kv);
411     System.err.println("kvFromKey=" + kvFromKey);
412     assertEquals(kvFromKey.toString(),
413         kv.toString().replaceAll("=[0-9]+", "=0"));
414   }
415 
416   /**
417    * Tests that getTimestamp() does always return the proper timestamp, even after updating it.
418    * See HBASE-6265.
419    */
420   public void testGetTimestamp() {
421     KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
422       Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP,
423       Bytes.toBytes("myValue"));
424     long time1 = kv.getTimestamp();
425     kv.updateLatestStamp(Bytes.toBytes(12345L));
426     long time2 = kv.getTimestamp();
427     assertEquals(HConstants.LATEST_TIMESTAMP, time1);
428     assertEquals(12345L, time2);
429   }
430 
431   public void testKVsWithTags() {
432     byte[] row = Bytes.toBytes("myRow");
433     byte[] cf = Bytes.toBytes("myCF");
434     byte[] q = Bytes.toBytes("myQualifier");
435     byte[] value = Bytes.toBytes("myValue");
436     byte[] metaValue1 = Bytes.toBytes("metaValue1");
437     byte[] metaValue2 = Bytes.toBytes("metaValue2");
438     KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, new Tag[] {
439       new Tag((byte) 1, metaValue1), new Tag((byte) 2, metaValue2)
440     });
441     assertTrue(kv.getTagsLength() > 0);
442     assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), row, 0,
443       row.length));
444     assertTrue(Bytes.equals(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), cf, 0,
445       cf.length));
446     assertTrue(Bytes.equals(kv.getQualifierArray(), kv.getQualifierOffset(),
447       kv.getQualifierLength(), q, 0, q.length));
448     assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), value, 0,
449       value.length));
450     List<Tag> tags = kv.getTags();
451     assertNotNull(tags);
452     assertEquals(2, tags.size());
453     boolean meta1Ok = false, meta2Ok = false;
454     for (Tag tag : tags) {
455       if (tag.getType() == (byte) 1) {
456         if (Bytes.equals(tag.getValue(), metaValue1)) {
457           meta1Ok = true;
458         }
459       } else {
460         if (Bytes.equals(tag.getValue(), metaValue2)) {
461           meta2Ok = true;
462         }
463       }
464     }
465     assertTrue(meta1Ok);
466     assertTrue(meta2Ok);
467     Iterator<Tag> tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
468       kv.getTagsLength());
469     assertTrue(tagItr.hasNext());
470     Tag next = tagItr.next();
471     assertEquals(10, next.getTagLength());
472     assertEquals((byte) 1, next.getType());
473     Bytes.equals(next.getValue(), metaValue1);
474     assertTrue(tagItr.hasNext());
475     next = tagItr.next();
476     assertEquals(10, next.getTagLength());
477     assertEquals((byte) 2, next.getType());
478     Bytes.equals(next.getValue(), metaValue2);
479     assertFalse(tagItr.hasNext());
480 
481     tagItr = CellUtil.tagsIterator(kv.getTagsArray(), kv.getTagsOffset(),
482       kv.getTagsLength());
483     assertTrue(tagItr.hasNext());
484     next = tagItr.next();
485     assertEquals(10, next.getTagLength());
486     assertEquals((byte) 1, next.getType());
487     Bytes.equals(next.getValue(), metaValue1);
488     assertTrue(tagItr.hasNext());
489     next = tagItr.next();
490     assertEquals(10, next.getTagLength());
491     assertEquals((byte) 2, next.getType());
492     Bytes.equals(next.getValue(), metaValue2);
493     assertFalse(tagItr.hasNext());
494   }
495 
496   public void testMetaKeyComparator() {
497     MetaComparator c = new KeyValue.MetaComparator();
498     long now = System.currentTimeMillis();
499 
500     KeyValue a = new KeyValue(Bytes.toBytes("table1"), now);
501     KeyValue b = new KeyValue(Bytes.toBytes("table2"), now);
502     assertTrue(c.compare(a, b) < 0);
503 
504     a = new KeyValue(Bytes.toBytes("table1,111"), now);
505     b = new KeyValue(Bytes.toBytes("table2"), now);
506     assertTrue(c.compare(a, b) < 0);
507 
508     a = new KeyValue(Bytes.toBytes("table1"), now);
509     b = new KeyValue(Bytes.toBytes("table2,111"), now);
510     assertTrue(c.compare(a, b) < 0);
511 
512     a = new KeyValue(Bytes.toBytes("table,111"), now);
513     b = new KeyValue(Bytes.toBytes("table,2222"), now);
514     assertTrue(c.compare(a, b) < 0);
515 
516     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
517     b = new KeyValue(Bytes.toBytes("table,2222"), now);
518     assertTrue(c.compare(a, b) < 0);
519 
520     a = new KeyValue(Bytes.toBytes("table,111"), now);
521     b = new KeyValue(Bytes.toBytes("table,2222.bbb"), now);
522     assertTrue(c.compare(a, b) < 0);
523 
524     a = new KeyValue(Bytes.toBytes("table,,aaaa"), now);
525     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
526     assertTrue(c.compare(a, b) < 0);
527 
528     a = new KeyValue(Bytes.toBytes("table,111,aaaa"), now);
529     b = new KeyValue(Bytes.toBytes("table,111,bbb"), now);
530     assertTrue(c.compare(a, b) < 0);
531 
532     a = new KeyValue(Bytes.toBytes("table,111,xxxx"), now);
533     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
534     assertTrue(c.compare(a, b) < 0);
535 
536     a = new KeyValue(Bytes.toBytes("table,111,11,xxx"), now);
537     b = new KeyValue(Bytes.toBytes("table,111,222,bbb"), now);
538     assertTrue(c.compare(a, b) < 0);
539   }
540 
541   public void testEqualsAndHashCode() {
542     KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
543         Bytes.toBytes("qualA"), Bytes.toBytes("1"));
544     KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
545         Bytes.toBytes("qualA"), Bytes.toBytes("2"));
546     // We set a different sequence id on kvA2 to demonstrate that the equals and hashCode also
547     // don't take this into account.
548     kvA2.setSequenceId(2);
549     KeyValue kvB = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"),
550         Bytes.toBytes("qualB"), Bytes.toBytes("1"));
551 
552     assertEquals(kvA1, kvA2);
553     assertNotEquals(kvA1, kvB);
554     assertEquals(kvA1.hashCode(), kvA2.hashCode());
555     assertNotEquals(kvA1.hashCode(), kvB.hashCode());
556   }
557 
558   public void testKeyValueSerialization() throws Exception {
559     KeyValue[] keyValues = new KeyValue[] {
560       new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
561         Bytes.toBytes("1")),
562       new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
563         Bytes.toBytes("2")),
564       new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
565         System.currentTimeMillis(), Bytes.toBytes("2"),
566         new Tag[] {
567           new Tag((byte) 120, "tagA"),
568           new Tag((byte) 121, Bytes.toBytes("tagB")) }),
569       new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"),
570         System.currentTimeMillis(), Bytes.toBytes("2"),
571         new Tag[] { new Tag((byte) 0, "tagA") }),
572       new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes(""),
573         Bytes.toBytes("1")) };
574     ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
575     for (KeyValue kv : keyValues) {
576       DataOutputStream os = new DataOutputStream(byteArrayOutputStream);
577       KeyValueUtil.oswrite(kv, os, true);
578     }
579     DataInputStream is =
580         new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()));
581     for (int i = 0; i < keyValues.length; i++) {
582       LOG.info("Case#" + i + ": deserialize the kv: " + keyValues[i]);
583       KeyValue destKv = KeyValueUtil.createKeyValueFromInputStream(is, true);
584       assertEquals(keyValues[i], destKv);
585       assertArrayEquals(CellUtil.cloneValue(keyValues[i]), CellUtil.cloneValue(destKv));
586       assertArrayEquals(CellUtil.cloneTags(keyValues[i]), CellUtil.cloneTags(destKv));
587     }
588   }
589 
590   @Test
591   public void testNullByteArrayKeyValueFailure() {
592     // can't add to testCheckKeyValueBytesFailureCase because it
593     // goes through the InputStream KeyValue API which can't produce a null buffer
594     try {
595       new KeyValue(null, 0, 0);
596     } catch (IllegalArgumentException iae) {
597       assertEquals("Invalid to have null byte array in KeyValue.", iae.getMessage());
598       return;
599     }
600     fail("Should have thrown an IllegalArgumentException when " +
601         "creating a KeyValue with a null buffer");
602   }
603 
604   private static class FailureCase {
605     byte[] buf;
606     int offset;
607     int length;
608     boolean withTags;
609     String expectedMessage;
610 
611     public FailureCase(byte[] buf, int offset, int length, boolean withTags,
612         String expectedMessage) {
613       this.buf = buf;
614       this.offset = offset;
615       this.length = length;
616       this.withTags = withTags;
617       this.expectedMessage = expectedMessage;
618     }
619 
620     @Override
621     public String toString() {
622       return "FailureCaseDetails: [buf=" + Bytes.toStringBinary(buf, offset, length) + ", offset="
623           + offset + ", " + "length=" + length + ", expectedMessage=" + expectedMessage
624           + ", withtags=" + withTags + "]";
625     }
626 
627     public String getExpectedMessage() {
628       return this.expectedMessage + KeyValueUtil.bytesToHex(buf, offset, length);
629     }
630   }
631 
632   @Test
633   public void testCheckKeyValueBytesFailureCase() throws Exception {
634     byte[][] inputs = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, // case.0
635       Bytes.toBytesBinary("a"), // case.1
636       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01"), // case.2
637       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00"), // case.3
638       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"), // case.4
639       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00"), // case.5
640       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x01"), // case.6
641       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW"), // case.7
642       Bytes.toBytesBinary("\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01"), // case.8
643       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\xFF"
644           + "\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\xFF\\x03"), // case.9
645       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
646           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x03"), // case.10
647       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
648           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04"), // case.11
649       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
650           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04VALUE"), // case.12
651     };
652     String[] outputs = new String[] { "Overflow when reading key length at position=0",
653       "Overflow when reading key length at position=0",
654       "Invalid key length in KeyValue. keyLength=1",
655       "Overflow when reading value length at position=4",
656       "Invalid value length in KeyValue, valueLength=1",
657       "Overflow when reading row length at position=8",
658       "Invalid row length in KeyValue, rowLength=1",
659       "Overflow when reading family length at position=13",
660       "Invalid family length in KeyValue, familyLength=1", "Timestamp cannot be negative, ts=-1",
661       "Invalid type in KeyValue, type=3", "Overflow when reading value part at position=25",
662       "Invalid tags length in KeyValue at position=26"};
663     byte[][] withTagsInputs = new byte[][] {
664       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
665           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x01"), // case.13
666       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
667           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x01"), // case.14
668       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
669           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x04\\x00\\x03\\x00A"), // case.15
670       // case.16
671       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
672           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0A\\x00\\x04\\x00TAG\\x00\\x04"
673           + "\\xFFT"),
674       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
675           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
676           + "\\xF0COME\\x00"), // case.17
677       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
678           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x0C\\x00\\x04\\x00TAG\\x00\\x05"
679           + "\\xF0COME"), // case.18
680       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
681           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x00"), // case.19
682       Bytes.toBytesBinary("\\x00\\x00\\x00\\x11\\x00\\x00\\x00\\x01\\x00\\x03ROW\\x01FQ\\x00"
683           + "\\x00\\x00\\x00\\x00\\x00\\x00\\x01\\x04V\\x00\\x1B\\x00\\x05\\x01TAG1\\x00\\x05"
684           + "\\x02TAG2\\x00\\x05\\x03TAG3\\x00\\x05\\x04TAG4"), // case.20
685     };
686     String[] withTagsOutputs = new String[] { "Overflow when reading tags length at position=26",
687       "Invalid tags length in KeyValue at position=26",
688       "Invalid tag length at position=28, tagLength=3",
689       "Invalid tag length at position=34, tagLength=4",
690       "Some redundant bytes in KeyValue's buffer, startOffset=41, endOffset=42", null, null,
691       null,
692     };
693     assertEquals(inputs.length, outputs.length);
694     assertEquals(withTagsInputs.length, withTagsOutputs.length);
695 
696     FailureCase[] cases = new FailureCase[inputs.length + withTagsInputs.length];
697     for (int i = 0; i < inputs.length; i++) {
698       cases[i] = new FailureCase(inputs[i], 0, inputs[i].length, false, outputs[i]);
699     }
700     for (int i = 0; i < withTagsInputs.length; i++) {
701       cases[inputs.length + i] =
702           new FailureCase(withTagsInputs[i], 0, withTagsInputs[i].length, true, withTagsOutputs[i]);
703     }
704 
705     for (int i = 0; i < cases.length; i++) {
706       FailureCase c = cases[i];
707       ByteArrayOutputStream baos = new ByteArrayOutputStream();
708       DataOutputStream os = new DataOutputStream(baos);
709       ByteBufferUtils.putInt(os, c.length);
710       os.write(c.buf, c.offset, c.length);
711       try {
712         KeyValueUtil.createKeyValueFromInputStream(
713           new DataInputStream(new ByteArrayInputStream(baos.toByteArray())), c.withTags);
714         if (c.expectedMessage != null) {
715           fail("Should fail when parse key value from an invalid bytes for case#" + i + ". " + c);
716         }
717       } catch (IllegalArgumentException e) {
718         assertEquals("Case#" + i + " failed," + c, c.getExpectedMessage(), e.getMessage());
719       }
720     }
721   }
722 
723   @Test
724   public void testRawBytesComparator() {
725     long ts = System.currentTimeMillis();
726     byte[] key = Bytes.toBytes("key");
727     byte[] cf = Bytes.toBytes("cf");
728     byte[] qualifier = Bytes.toBytes("qualifier");
729     byte[] value = Bytes.toBytes("value");
730     KeyValue kvA1 = new KeyValue(key, cf, qualifier, ts, Type.Put, value);
731     KeyValue kvA2 = new KeyValue(key, cf, qualifier, ts, Type.DeleteFamily, value);
732     assertTrue(KeyValue.RAW_COMPARATOR.compare(kvA1, kvA2) > 0);
733   }
734 }