View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.NavigableMap;
26  import java.util.TreeMap;
27  
28  import org.apache.hadoop.hbase.KeyValue.Type;
29  import org.apache.hadoop.hbase.testclassification.SmallTests;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.junit.Assert;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  
35  @Category(SmallTests.class)
36  public class TestCellUtil {
37    /**
38     * CellScannable used in test. Returns a {@link TestCellScanner}
39     */
40    private static class TestCellScannable implements CellScannable {
41      private final int cellsCount;
42      TestCellScannable(final int cellsCount) {
43        this.cellsCount = cellsCount;
44      }
45      @Override
46      public CellScanner cellScanner() {
47        return new TestCellScanner(this.cellsCount);
48      }
49    }
50  
51    /**
52     * CellScanner used in test.
53     */
54    private static class TestCellScanner implements CellScanner {
55      private int count = 0;
56      private Cell current = null;
57      private final int cellsCount;
58  
59      TestCellScanner(final int cellsCount) {
60        this.cellsCount = cellsCount;
61      }
62  
63      @Override
64      public Cell current() {
65        return this.current;
66      }
67  
68      @Override
69      public boolean advance() {
70        if (this.count < cellsCount) {
71          this.current = new TestCell(this.count);
72          this.count++;
73          return true;
74        }
75        return false;
76      }
77    }
78  
79    /**
80     * Cell used in test. Has row only.
81     */
82    private static class TestCell implements Cell {
83      private final byte [] row;
84  
85      TestCell(final int i) {
86        this.row = Bytes.toBytes(i);
87      }
88  
89      @Override
90      public byte[] getRowArray() {
91        return this.row;
92      }
93  
94      @Override
95      public int getRowOffset() {
96        return 0;
97      }
98  
99      @Override
100     public short getRowLength() {
101       return (short)this.row.length;
102     }
103 
104     @Override
105     public byte[] getFamilyArray() {
106       // TODO Auto-generated method stub
107       return null;
108     }
109 
110     @Override
111     public int getFamilyOffset() {
112       // TODO Auto-generated method stub
113       return 0;
114     }
115 
116     @Override
117     public byte getFamilyLength() {
118       // TODO Auto-generated method stub
119       return 0;
120     }
121 
122     @Override
123     public byte[] getQualifierArray() {
124       // TODO Auto-generated method stub
125       return null;
126     }
127 
128     @Override
129     public int getQualifierOffset() {
130       // TODO Auto-generated method stub
131       return 0;
132     }
133 
134     @Override
135     public int getQualifierLength() {
136       // TODO Auto-generated method stub
137       return 0;
138     }
139 
140     @Override
141     public long getTimestamp() {
142       // TODO Auto-generated method stub
143       return 0;
144     }
145 
146     @Override
147     public byte getTypeByte() {
148       // TODO Auto-generated method stub
149       return 0;
150     }
151 
152     @Override
153     public long getMvccVersion() {
154       // TODO Auto-generated method stub
155       return 0;
156     }
157 
158     @Override
159     public byte[] getValueArray() {
160       // TODO Auto-generated method stub
161       return null;
162     }
163 
164     @Override
165     public int getValueOffset() {
166       // TODO Auto-generated method stub
167       return 0;
168     }
169 
170     @Override
171     public int getValueLength() {
172       // TODO Auto-generated method stub
173       return 0;
174     }
175 
176     @Override
177     public byte[] getTagsArray() {
178       // TODO Auto-generated method stub
179       return null;
180     }
181 
182     @Override
183     public int getTagsOffset() {
184       // TODO Auto-generated method stub
185       return 0;
186     }
187 
188     @Override
189     public byte[] getValue() {
190       // TODO Auto-generated method stub
191       return null;
192     }
193 
194     @Override
195     public byte[] getFamily() {
196       // TODO Auto-generated method stub
197       return null;
198     }
199 
200     @Override
201     public byte[] getQualifier() {
202       // TODO Auto-generated method stub
203       return null;
204     }
205 
206     @Override
207     public byte[] getRow() {
208       // TODO Auto-generated method stub
209       return null;
210     }
211 
212     @Override
213     public long getSequenceId() {
214       // TODO Auto-generated method stub
215       return 0;
216     }
217 
218     @Override
219     public int getTagsLength() {
220       // TODO Auto-generated method stub
221       return 0;
222     }
223   }
224 
225   /**
226    * Was overflowing if 100k or so lists of cellscanners to return.
227    */
228   @Test
229   public void testCreateCellScannerOverflow() throws IOException {
230     consume(doCreateCellScanner(1, 1), 1);
231     consume(doCreateCellScanner(3, 0), 0);
232     consume(doCreateCellScanner(3, 3), 3 * 3);
233     consume(doCreateCellScanner(0, 1), 0);
234     // Do big number. See HBASE-11813 for why.
235     final int hundredK = 100000;
236     consume(doCreateCellScanner(hundredK, 0), 0);
237     consume(doCreateCellArray(1), 1);
238     consume(doCreateCellArray(0), 0);
239     consume(doCreateCellArray(3), 3);
240     List<CellScannable> cells = new ArrayList<>(hundredK);
241     for (int i = 0; i < hundredK; i++) {
242       cells.add(new TestCellScannable(1));
243     }
244     consume(CellUtil.createCellScanner(cells), hundredK);
245     NavigableMap<byte [], List<Cell>> m = new TreeMap<>(Bytes.BYTES_COMPARATOR);
246     List<Cell> cellArray = new ArrayList<>(hundredK);
247     for (int i = 0; i < hundredK; i++) {
248       cellArray.add(new TestCell(i));
249     }
250     m.put(new byte [] {'f'}, cellArray);
251     consume(CellUtil.createCellScanner(m), hundredK);
252   }
253 
254   private CellScanner doCreateCellArray(final int itemsPerList) {
255     Cell [] cells = new Cell [itemsPerList];
256     for (int i = 0; i < itemsPerList; i++) {
257       cells[i] = new TestCell(i);
258     }
259     return CellUtil.createCellScanner(cells);
260   }
261 
262   private CellScanner doCreateCellScanner(final int listsCount, final int itemsPerList) {
263     List<CellScannable> cells = new ArrayList<>(listsCount);
264     for (int i = 0; i < listsCount; i++) {
265       CellScannable cs = new CellScannable() {
266         @Override
267         public CellScanner cellScanner() {
268           return new TestCellScanner(itemsPerList);
269         }
270       };
271       cells.add(cs);
272     }
273     return CellUtil.createCellScanner(cells);
274   }
275 
276   private void consume(final CellScanner scanner, final int expected) throws IOException {
277     int count = 0;
278     while (scanner.advance()) {
279       count++;
280     }
281     Assert.assertEquals(expected, count);
282   }
283 
284   @Test
285   public void testOverlappingKeys() {
286     byte[] empty = HConstants.EMPTY_BYTE_ARRAY;
287     byte[] a = Bytes.toBytes("a");
288     byte[] b = Bytes.toBytes("b");
289     byte[] c = Bytes.toBytes("c");
290     byte[] d = Bytes.toBytes("d");
291 
292     // overlaps
293     Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, b));
294     Assert.assertTrue(CellUtil.overlappingKeys(a, c, a, b));
295     Assert.assertTrue(CellUtil.overlappingKeys(a, b, a, c));
296     Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, c));
297     Assert.assertTrue(CellUtil.overlappingKeys(a, c, b, c));
298     Assert.assertTrue(CellUtil.overlappingKeys(a, d, b, c));
299     Assert.assertTrue(CellUtil.overlappingKeys(b, c, a, d));
300 
301     Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, b));
302     Assert.assertTrue(CellUtil.overlappingKeys(empty, b, a, c));
303 
304     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, b));
305     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, c));
306 
307     Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, b));
308     Assert.assertTrue(CellUtil.overlappingKeys(a, empty, a, c));
309 
310     Assert.assertTrue(CellUtil.overlappingKeys(a, b, empty, empty));
311     Assert.assertTrue(CellUtil.overlappingKeys(empty, empty, a, b));
312 
313     // non overlaps
314     Assert.assertFalse(CellUtil.overlappingKeys(a, b, c, d));
315     Assert.assertFalse(CellUtil.overlappingKeys(c, d, a, b));
316 
317     Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, d));
318     Assert.assertFalse(CellUtil.overlappingKeys(b, c, c, empty));
319     Assert.assertFalse(CellUtil.overlappingKeys(b, c, d, empty));
320     Assert.assertFalse(CellUtil.overlappingKeys(c, d, b, c));
321     Assert.assertFalse(CellUtil.overlappingKeys(c, empty, b, c));
322     Assert.assertFalse(CellUtil.overlappingKeys(d, empty, b, c));
323 
324     Assert.assertFalse(CellUtil.overlappingKeys(b, c, a, b));
325     Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, b));
326     Assert.assertFalse(CellUtil.overlappingKeys(b, c, empty, a));
327     Assert.assertFalse(CellUtil.overlappingKeys(a,b, b, c));
328     Assert.assertFalse(CellUtil.overlappingKeys(empty, b, b, c));
329     Assert.assertFalse(CellUtil.overlappingKeys(empty, a, b, c));
330   }
331 
332   @Test
333   public void testFindCommonPrefixInFlatKey() {
334     // The whole key matching case
335     KeyValue kv1 = new KeyValue(Bytes.toBytes("r1"), Bytes.toBytes("f1"),
336         Bytes.toBytes("q1"), null);
337     Assert.assertEquals(kv1.getKeyLength(),
338         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, true));
339     Assert.assertEquals(kv1.getKeyLength(),
340         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, false, true));
341     Assert.assertEquals(kv1.getKeyLength() - KeyValue.TIMESTAMP_TYPE_SIZE,
342         CellUtil.findCommonPrefixInFlatKey(kv1, kv1, true, false));
343     // The rk length itself mismatch
344     KeyValue kv2 = new KeyValue(Bytes.toBytes("r12"), Bytes.toBytes("f1"),
345         Bytes.toBytes("q1"), null);
346     Assert.assertEquals(1, CellUtil.findCommonPrefixInFlatKey(kv1, kv2, true, true));
347     // part of rk is same
348     KeyValue kv3 = new KeyValue(Bytes.toBytes("r14"), Bytes.toBytes("f1"),
349         Bytes.toBytes("q1"), null);
350     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + Bytes.toBytes("r1").length,
351         CellUtil.findCommonPrefixInFlatKey(kv2, kv3, true, true));
352     // entire rk is same but different cf name
353     KeyValue kv4 = new KeyValue(Bytes.toBytes("r14"), Bytes.toBytes("f2"),
354         Bytes.toBytes("q1"), null);
355     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv3.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
356         + Bytes.toBytes("f").length,
357         CellUtil.findCommonPrefixInFlatKey(kv3, kv4, false, true));
358     // rk and family are same and part of qualifier
359     KeyValue kv5 = new KeyValue(Bytes.toBytes("r14"), Bytes.toBytes("f2"),
360         Bytes.toBytes("q123"), null);
361     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv3.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
362         + kv4.getFamilyLength() + kv4.getQualifierLength(),
363         CellUtil.findCommonPrefixInFlatKey(kv4, kv5, true, true));
364     // rk, cf and q are same. ts differs
365     KeyValue kv6 = new KeyValue(Bytes.toBytes("rk"), 1234L);
366     KeyValue kv7 = new KeyValue(Bytes.toBytes("rk"), 1235L);
367     // only last byte out of 8 ts bytes in ts part differs
368     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
369         + kv6.getFamilyLength() + kv6.getQualifierLength() + 7,
370         CellUtil.findCommonPrefixInFlatKey(kv6, kv7, true, true));
371     // rk, cf, q and ts are same. Only type differs
372     KeyValue kv8 = new KeyValue(Bytes.toBytes("rk"), 1234L, Type.Delete);
373     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
374         + kv6.getFamilyLength() + kv6.getQualifierLength() + KeyValue.TIMESTAMP_SIZE,
375         CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, true));
376     // With out TS_TYPE check
377     Assert.assertEquals(KeyValue.ROW_LENGTH_SIZE + kv6.getRowLength() + KeyValue.FAMILY_LENGTH_SIZE
378         + kv6.getFamilyLength() + kv6.getQualifierLength(),
379         CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, false));
380   }
381 
382   /**
383    * Assert CellUtil makes Cell toStrings same way we do KeyValue toStrings.
384    */
385   @Test
386   public void testToString() {
387     byte [] row = Bytes.toBytes("row");
388     long ts = 123L;
389     // Make a KeyValue and a Cell and see if same toString result.
390     KeyValue kv = new KeyValue(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
391         ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY);
392     Cell cell = CellUtil.createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY,
393         ts, KeyValue.Type.Minimum.getCode(), HConstants.EMPTY_BYTE_ARRAY);
394     String cellToString = CellUtil.getCellKeyAsString(cell);
395     assertEquals(kv.toString(), cellToString);
396     // Do another w/ non-null family.
397     byte [] f = new byte [] {'f'};
398     byte [] q = new byte [] {'q'};
399     kv = new KeyValue(row, f, q, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY);
400     cell = CellUtil.createCell(row, f, q, ts, KeyValue.Type.Minimum.getCode(),
401         HConstants.EMPTY_BYTE_ARRAY);
402     cellToString = CellUtil.getCellKeyAsString(cell);
403     assertEquals(kv.toString(), cellToString);
404   }
405 
406   @Test
407   public void testToString1() {
408     String row = "test.row";
409     String family = "test.family";
410     String qualifier = "test.qualifier";
411     long timestamp = 42;
412     Type type = Type.Put;
413     String value = "test.value";
414     long seqId = 1042;
415 
416     Cell cell = CellUtil.createCell(Bytes.toBytes(row), Bytes.toBytes(family),
417       Bytes.toBytes(qualifier), timestamp, type.getCode(), Bytes.toBytes(value), seqId);
418 
419     String nonVerbose = CellUtil.toString(cell, false);
420     String verbose = CellUtil.toString(cell, true);
421 
422     System.out.println("nonVerbose=" + nonVerbose);
423     System.out.println("verbose=" + verbose);
424 
425     Assert.assertEquals(
426         String.format("%s/%s:%s/%d/%s/vlen=%s/seqid=%s",
427           row, family, qualifier, timestamp, type.toString(),
428           Bytes.toBytes(value).length, seqId),
429         nonVerbose);
430 
431     Assert.assertEquals(
432       String.format("%s/%s:%s/%d/%s/vlen=%s/seqid=%s/%s",
433         row, family, qualifier, timestamp, type.toString(), Bytes.toBytes(value).length,
434         seqId, value),
435       verbose);
436 
437     // TODO: test with tags
438   }
439 }