1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.client;
18
19 import static org.apache.hadoop.hbase.HConstants.RPC_CODEC_CONF_KEY;
20 import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_CODEC_CLASS;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.hbase.Cell;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HRegionLocation;
40 import org.apache.hadoop.hbase.HTestConst;
41 import org.apache.hadoop.hbase.KeyValue;
42 import org.apache.hadoop.hbase.exceptions.DeserializationException;
43 import org.apache.hadoop.hbase.filter.FilterBase;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.MiniHBaseCluster;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
48 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
49 import org.apache.hadoop.hbase.master.HMaster;
50 import org.apache.hadoop.hbase.master.RegionState.State;
51 import org.apache.hadoop.hbase.master.RegionStates;
52 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
53 import org.apache.hadoop.hbase.regionserver.HRegionServer;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.ConfigUtil;
56 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
57 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
58 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
59 import org.junit.After;
60 import org.junit.AfterClass;
61 import org.junit.Before;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65
66
67
68
69 @Category(MediumTests.class)
70 public class TestScannersFromClientSide {
71 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
72
73 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
74 private static byte [] ROW = Bytes.toBytes("testRow");
75 private static byte [] FAMILY = Bytes.toBytes("testFamily");
76 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
77 private static byte [] VALUE = Bytes.toBytes("testValue");
78
79
80
81
82 @BeforeClass
83 public static void setUpBeforeClass() throws Exception {
84 Configuration conf = TEST_UTIL.getConfiguration();
85 conf.setLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, 10 * 1024 * 1024);
86 TEST_UTIL.startMiniCluster(3);
87 }
88
89
90
91
92 @AfterClass
93 public static void tearDownAfterClass() throws Exception {
94 TEST_UTIL.shutdownMiniCluster();
95 }
96
97
98
99
100 @Before
101 public void setUp() throws Exception {
102
103 }
104
105
106
107
108 @After
109 public void tearDown() throws Exception {
110
111 }
112
113
114
115
116
117
118 @Test
119 public void testScanBatch() throws Exception {
120 TableName TABLE = TableName.valueOf("testScanBatch");
121 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
122
123 Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
124
125 Put put;
126 Scan scan;
127 Delete delete;
128 Result result;
129 ResultScanner scanner;
130 boolean toLog = true;
131 List<Cell> kvListExp;
132
133
134 put = new Put(ROW);
135 for (int i=0; i < QUALIFIERS.length; i++) {
136 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
137 put.add(kv);
138 }
139 ht.put(put);
140
141
142 put = new Put(ROW);
143 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
144 put.add(kv);
145 ht.put(put);
146
147
148 delete = new Delete(ROW);
149 delete.deleteFamily(FAMILY, 3);
150 ht.delete(delete);
151
152
153 scan = new Scan().withStartRow(ROW);
154 scan.setMaxVersions();
155 scanner = ht.getScanner(scan);
156
157
158 kvListExp = new ArrayList<Cell>();
159 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
160 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
161 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
162 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
163 result = scanner.next();
164 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
165
166
167 scan = new Scan().withStartRow(ROW);
168 scan.setMaxVersions();
169 scan.setBatch(2);
170 scanner = ht.getScanner(scan);
171
172
173 kvListExp = new ArrayList<Cell>();
174 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
175 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
176 result = scanner.next();
177 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
178
179
180 kvListExp = new ArrayList<Cell>();
181 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
182 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
183 result = scanner.next();
184 verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
185
186 }
187
188 @Test
189 public void testMaxResultSizeIsSetToDefault() throws Exception {
190 TableName TABLE = TableName.valueOf("testMaxResultSizeIsSetToDefault");
191 Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
192
193
194 long expectedMaxResultSize =
195 TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
196 HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
197
198 int numRows = 5;
199 byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows);
200
201 int numQualifiers = 10;
202 byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers);
203
204
205
206
207 int cellSize = (int) (expectedMaxResultSize / (numQualifiers - 1));
208 byte[] cellValue = Bytes.createMaxByteArray(cellSize);
209
210 Put put;
211 List<Put> puts = new ArrayList<Put>();
212 for (int row = 0; row < ROWS.length; row++) {
213 put = new Put(ROWS[row]);
214 for (int qual = 0; qual < QUALIFIERS.length; qual++) {
215 KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue);
216 put.add(kv);
217 }
218 puts.add(put);
219 }
220 ht.put(puts);
221
222
223 Scan scan = new Scan();
224
225 ResultScanner scanner = ht.getScanner(scan);
226 assertTrue(scanner instanceof ClientScanner);
227 ClientScanner clientScanner = (ClientScanner) scanner;
228
229
230 scanner.next();
231
232
233
234 assertTrue("The cache contains: " + clientScanner.getCacheSize() + " results",
235 clientScanner.getCacheSize() <= 1);
236 }
237
238 @Test
239 public void testSmallScan() throws Exception {
240 TableName TABLE = TableName.valueOf("testSmallScan");
241
242 int numRows = 10;
243 byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows);
244
245 int numQualifiers = 10;
246 byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers);
247
248 Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
249
250 Put put;
251 List<Put> puts = new ArrayList<Put>();
252 for (int row = 0; row < ROWS.length; row++) {
253 put = new Put(ROWS[row]);
254 for (int qual = 0; qual < QUALIFIERS.length; qual++) {
255 KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE);
256 put.add(kv);
257 }
258 puts.add(put);
259 }
260 ht.put(puts);
261
262 int expectedRows = numRows;
263 int expectedCols = numRows * numQualifiers;
264
265
266 testSmallScan(ht, true, expectedRows, expectedCols);
267 testSmallScan(ht, false, expectedRows, expectedCols);
268 }
269
270
271
272
273
274
275
276
277
278 private void testSmallScan(Table table, boolean reversed, int rows, int columns) throws Exception {
279 Scan baseScan = new Scan();
280 baseScan.setReversed(reversed);
281 baseScan.setSmall(true);
282
283 Scan scan = new Scan(baseScan);
284 verifyExpectedCounts(table, scan, rows, columns);
285
286 scan = new Scan(baseScan);
287 scan.setMaxResultSize(1);
288 verifyExpectedCounts(table, scan, rows, columns);
289
290 scan = new Scan(baseScan);
291 scan.setMaxResultSize(1);
292 scan.setCaching(Integer.MAX_VALUE);
293 verifyExpectedCounts(table, scan, rows, columns);
294 }
295
296 private void verifyExpectedCounts(Table table, Scan scan, int expectedRowCount,
297 int expectedCellCount) throws Exception {
298 ResultScanner scanner = table.getScanner(scan);
299
300 int rowCount = 0;
301 int cellCount = 0;
302 Result r = null;
303 while ((r = scanner.next()) != null) {
304 rowCount++;
305 cellCount += r.rawCells().length;
306 }
307
308 assertTrue("Expected row count: " + expectedRowCount + " Actual row count: " + rowCount,
309 expectedRowCount == rowCount);
310 assertTrue("Expected cell count: " + expectedCellCount + " Actual cell count: " + cellCount,
311 expectedCellCount == cellCount);
312 scanner.close();
313 }
314
315
316
317
318
319
320 @Test
321 public void testGetMaxResults() throws Exception {
322 byte [] TABLE = Bytes.toBytes("testGetMaxResults");
323 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
324 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
325
326 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
327
328 Get get;
329 Put put;
330 Result result;
331 boolean toLog = true;
332 List<Cell> kvListExp;
333
334 kvListExp = new ArrayList<Cell>();
335
336 put = new Put(ROW);
337 for (int i=0; i < 10; i++) {
338 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
339 put.add(kv);
340 kvListExp.add(kv);
341 }
342 ht.put(put);
343
344 get = new Get(ROW);
345 result = ht.get(get);
346 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
347
348 get = new Get(ROW);
349 get.setMaxResultsPerColumnFamily(2);
350 result = ht.get(get);
351 kvListExp = new ArrayList<Cell>();
352 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
353 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
354 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
355
356
357 get = new Get(ROW);
358 get.setMaxResultsPerColumnFamily(5);
359 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
360 true));
361 result = ht.get(get);
362 kvListExp = new ArrayList<Cell>();
363 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
364 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
365 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
366 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
367 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
368
369
370
371 put = new Put(ROW);
372 for (int i=0; i < QUALIFIERS.length; i++) {
373 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
374 put.add(kv);
375 }
376 ht.put(put);
377
378 put = new Put(ROW);
379 for (int i=0; i < 10; i++) {
380 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
381 put.add(kv);
382 }
383 ht.put(put);
384
385 get = new Get(ROW);
386 get.setMaxResultsPerColumnFamily(12);
387 get.addFamily(FAMILIES[1]);
388 get.addFamily(FAMILIES[2]);
389 result = ht.get(get);
390 kvListExp = new ArrayList<Cell>();
391
392 for (int i=0; i < 10; i++) {
393 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
394 }
395 for (int i=0; i < 2; i++) {
396 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
397 }
398 for (int i=10; i < 20; i++) {
399 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
400 }
401 verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
402
403
404 get = new Get(ROW);
405 get.setMaxResultsPerColumnFamily(3);
406 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
407 result = ht.get(get);
408 kvListExp = new ArrayList<Cell>();
409 for (int i=2; i < 5; i++) {
410 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
411 }
412 for (int i=2; i < 5; i++) {
413 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
414 }
415 for (int i=2; i < 5; i++) {
416 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
417 }
418 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
419
420 get = new Get(ROW);
421 get.setMaxResultsPerColumnFamily(7);
422 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
423 result = ht.get(get);
424 kvListExp = new ArrayList<Cell>();
425 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
426 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
427 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
428 for (int i=10; i < 16; i++) {
429 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
430 }
431 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
432
433 }
434
435
436
437
438
439
440 @Test
441 public void testScanMaxResults() throws Exception {
442 byte [] TABLE = Bytes.toBytes("testScanLimit");
443 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
444 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
445 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
446
447 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
448
449 Put put;
450 Scan scan;
451 Result result;
452 boolean toLog = true;
453 List<Cell> kvListExp, kvListScan;
454
455 kvListExp = new ArrayList<Cell>();
456
457 for (int r=0; r < ROWS.length; r++) {
458 put = new Put(ROWS[r]);
459 for (int c=0; c < FAMILIES.length; c++) {
460 for (int q=0; q < QUALIFIERS.length; q++) {
461 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
462 put.add(kv);
463 if (q < 4) {
464 kvListExp.add(kv);
465 }
466 }
467 }
468 ht.put(put);
469 }
470
471 scan = new Scan();
472 scan.setMaxResultsPerColumnFamily(4);
473 ResultScanner scanner = ht.getScanner(scan);
474 kvListScan = new ArrayList<Cell>();
475 while ((result = scanner.next()) != null) {
476 for (Cell kv : result.listCells()) {
477 kvListScan.add(kv);
478 }
479 }
480 result = Result.create(kvListScan);
481 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
482
483 }
484
485
486
487
488
489
490 @Test
491 public void testGetRowOffset() throws Exception {
492 byte [] TABLE = Bytes.toBytes("testGetRowOffset");
493 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
494 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
495
496 Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
497
498 Get get;
499 Put put;
500 Result result;
501 boolean toLog = true;
502 List<Cell> kvListExp;
503
504
505 kvListExp = new ArrayList<Cell>();
506 put = new Put(ROW);
507 for (int i=0; i < 10; i++) {
508 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
509 put.add(kv);
510
511 if (i < 2) continue;
512 kvListExp.add(kv);
513 }
514 ht.put(put);
515
516
517 get = new Get(ROW);
518 get.setRowOffsetPerColumnFamily(2);
519 result = ht.get(get);
520 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
521
522
523 get = new Get(ROW);
524 get.setRowOffsetPerColumnFamily(20);
525 result = ht.get(get);
526 kvListExp = new ArrayList<Cell>();
527 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
528
529
530 get = new Get(ROW);
531 get.setRowOffsetPerColumnFamily(4);
532 get.setMaxResultsPerColumnFamily(5);
533 result = ht.get(get);
534 kvListExp = new ArrayList<Cell>();
535 for (int i=4; i < 9; i++) {
536 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
537 }
538 verifyResult(result, kvListExp, toLog,
539 "Testing offset + setMaxResultsPerCF");
540
541
542 get = new Get(ROW);
543 get.setRowOffsetPerColumnFamily(1);
544 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
545 true));
546 result = ht.get(get);
547 kvListExp = new ArrayList<Cell>();
548 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
549 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
550 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
551 verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
552
553
554
555 for(int j=2; j > 0; j--) {
556 put = new Put(ROW);
557 for (int i=0; i < 10; i++) {
558 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
559 put.add(kv);
560 }
561 ht.put(put);
562 }
563
564 get = new Get(ROW);
565 get.setRowOffsetPerColumnFamily(4);
566 get.setMaxResultsPerColumnFamily(2);
567 get.addFamily(FAMILIES[1]);
568 get.addFamily(FAMILIES[2]);
569 result = ht.get(get);
570 kvListExp = new ArrayList<Cell>();
571
572 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
573 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
574 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
575 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
576 verifyResult(result, kvListExp, toLog,
577 "Testing offset + multiple CFs + maxResults");
578 }
579
580 @Test
581 public void testScanRawDeleteFamilyVersion() throws Exception {
582 TableName tableName = TableName.valueOf("testScanRawDeleteFamilyVersion");
583 TEST_UTIL.createTable(tableName, FAMILY);
584 Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
585 conf.set(RPC_CODEC_CONF_KEY, "");
586 conf.set(DEFAULT_CODEC_CLASS, "");
587 try (Connection connection = ConnectionFactory.createConnection(conf);
588 Table table = connection.getTable(tableName)) {
589 Delete delete = new Delete(ROW);
590 delete.addFamilyVersion(FAMILY, 0L);
591 table.delete(delete);
592 Scan scan = new Scan().withStartRow(ROW).setRaw(true);
593 ResultScanner scanner = table.getScanner(scan);
594 int count = 0;
595 while (scanner.next() != null) {
596 count++;
597 }
598 assertEquals(1, count);
599 } finally {
600 TEST_UTIL.deleteTable(tableName);
601 }
602 }
603
604
605
606
607
608
609
610 @Test
611 public void testScanOnReopenedRegion() throws Exception {
612 TableName TABLE = TableName.valueOf("testScanOnReopenedRegion");
613 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
614
615 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
616
617 Put put;
618 Scan scan;
619 Result result;
620 ResultScanner scanner;
621 boolean toLog = false;
622 List<Cell> kvListExp;
623
624
625 put = new Put(ROW);
626 for (int i=0; i < QUALIFIERS.length; i++) {
627 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
628 put.add(kv);
629 }
630 ht.put(put);
631
632 scan = new Scan().withStartRow(ROW);
633 scanner = ht.getScanner(scan);
634
635 HRegionLocation loc = ht.getRegionLocation(ROW);
636 HRegionInfo hri = loc.getRegionInfo();
637 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
638 byte[] regionName = hri.getRegionName();
639 int i = cluster.getServerWith(regionName);
640 HRegionServer rs = cluster.getRegionServer(i);
641 ProtobufUtil.closeRegion(null,
642 rs.getRSRpcServices(), rs.getServerName(), regionName, false);
643 long startTime = EnvironmentEdgeManager.currentTime();
644 long timeOut = 300000;
645 while (true) {
646 if (rs.getOnlineRegion(regionName) == null) {
647 break;
648 }
649 assertTrue("Timed out in closing the testing region",
650 EnvironmentEdgeManager.currentTime() < startTime + timeOut);
651 Thread.sleep(500);
652 }
653
654
655 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
656 try {
657 HMaster master = cluster.getMaster();
658 RegionStates states = master.getAssignmentManager().getRegionStates();
659 states.regionOffline(hri);
660 states.updateRegionState(hri, State.OPENING);
661 if (ConfigUtil.useZKForAssignment(TEST_UTIL.getConfiguration())) {
662 ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
663 }
664 ProtobufUtil.openRegion(null, rs.getRSRpcServices(), rs.getServerName(), hri);
665 startTime = EnvironmentEdgeManager.currentTime();
666 while (true) {
667 if (rs.getOnlineRegion(regionName) != null) {
668 break;
669 }
670 assertTrue("Timed out in open the testing region",
671 EnvironmentEdgeManager.currentTime() < startTime + timeOut);
672 Thread.sleep(500);
673 }
674 } finally {
675 ZKAssign.deleteNodeFailSilent(zkw, hri);
676 }
677
678
679 kvListExp = new ArrayList<Cell>();
680 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
681 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
682 result = scanner.next();
683 verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
684 }
685
686 static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
687 String msg) {
688
689 LOG.info(msg);
690 LOG.info("Expected count: " + expKvList.size());
691 LOG.info("Actual count: " + result.size());
692 if (expKvList.size() == 0)
693 return;
694
695 int i = 0;
696 for (Cell kv : result.rawCells()) {
697 if (i >= expKvList.size()) {
698 break;
699 }
700
701 Cell kvExp = expKvList.get(i++);
702 if (toLog) {
703 LOG.info("get kv is: " + kv.toString());
704 LOG.info("exp kv is: " + kvExp.toString());
705 }
706 assertTrue("Not equal", kvExp.equals(kv));
707 }
708
709 assertEquals(expKvList.size(), result.size());
710 }
711
712 @Test
713 public void testReadExpiredDataForRawScan() throws IOException {
714 TableName tableName = TableName.valueOf("testReadExpiredDataForRawScan");
715 long ts = System.currentTimeMillis() - 10000;
716 byte[] value = Bytes.toBytes("expired");
717 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
718 table.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, ts, value));
719 assertArrayEquals(value, table.get(new Get(ROW)).getValue(FAMILY, QUALIFIER));
720 TEST_UTIL.getHBaseAdmin().modifyColumn(tableName,
721 new HColumnDescriptor(FAMILY).setTimeToLive(5));
722 try (ResultScanner scanner = table.getScanner(FAMILY)) {
723 assertNull(scanner.next());
724 }
725 try (ResultScanner scanner = table.getScanner(new Scan().setRaw(true))) {
726 assertArrayEquals(value, scanner.next().getValue(FAMILY, QUALIFIER));
727 assertNull(scanner.next());
728 }
729 }
730 }
731
732 @Test
733 public void testScanWithSameStartRowStopRow() throws IOException {
734 TableName tableName = TableName.valueOf("testScanWithSameStartRowStopRow");
735 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
736 table.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE));
737
738 Scan scan = new Scan().withStartRow(ROW).withStopRow(ROW);
739 try (ResultScanner scanner = table.getScanner(scan)) {
740 assertNull(scanner.next());
741 }
742
743 scan = new Scan().withStartRow(ROW, true).withStopRow(ROW, true);
744 try (ResultScanner scanner = table.getScanner(scan)) {
745 Result result = scanner.next();
746 assertNotNull(result);
747 assertArrayEquals(ROW, result.getRow());
748 assertArrayEquals(VALUE, result.getValue(FAMILY, QUALIFIER));
749 assertNull(scanner.next());
750 }
751
752 scan = new Scan().withStartRow(ROW, true).withStopRow(ROW, false);
753 try (ResultScanner scanner = table.getScanner(scan)) {
754 assertNull(scanner.next());
755 }
756
757 scan = new Scan().withStartRow(ROW, false).withStopRow(ROW, false);
758 try (ResultScanner scanner = table.getScanner(scan)) {
759 assertNull(scanner.next());
760 }
761
762 scan = new Scan().withStartRow(ROW, false).withStopRow(ROW, true);
763 try (ResultScanner scanner = table.getScanner(scan)) {
764 assertNull(scanner.next());
765 }
766 }
767 }
768
769 @Test
770 public void testScannerWithPartialResults() throws Exception {
771 TableName tableName = TableName.valueOf("testScannerWithPartialResults");
772 try (Table table = TEST_UTIL.createMultiRegionTable(tableName,
773 Bytes.toBytes("c"), 4)) {
774 List<Put> puts = new ArrayList<>();
775 byte[] largeArray = new byte[10000];
776 Put put = new Put(Bytes.toBytes("aaaa0"));
777 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("1"), Bytes.toBytes("1"));
778 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("2"), Bytes.toBytes("2"));
779 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("3"), Bytes.toBytes("3"));
780 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("4"), Bytes.toBytes("4"));
781 puts.add(put);
782 put = new Put(Bytes.toBytes("aaaa1"));
783 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("1"), Bytes.toBytes("1"));
784 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("2"), largeArray);
785 put.addColumn(Bytes.toBytes("c"), Bytes.toBytes("3"), largeArray);
786 puts.add(put);
787 table.put(puts);
788 Scan scan = new Scan();
789 scan.addFamily(Bytes.toBytes("c"));
790 scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, tableName.getName());
791 scan.setMaxResultSize(10001);
792 scan.setStopRow(Bytes.toBytes("bbbb"));
793 scan.setFilter(new LimitKVsReturnFilter());
794 ResultScanner rs = table.getScanner(scan);
795 Result result;
796 int expectedKvNumber = 6;
797 int returnedKvNumber = 0;
798 while((result = rs.next()) != null) {
799 returnedKvNumber += result.listCells().size();
800 }
801 rs.close();
802 assertEquals(expectedKvNumber, returnedKvNumber);
803 }
804 }
805
806 public static class LimitKVsReturnFilter extends FilterBase {
807
808 private int cellCount = 0;
809
810 @Override
811 public ReturnCode filterKeyValue(Cell v) throws IOException {
812 if (cellCount >= 6) {
813 cellCount++;
814 return ReturnCode.SKIP;
815 }
816 cellCount++;
817 return ReturnCode.INCLUDE;
818 }
819
820 @Override
821 public boolean filterAllRemaining() throws IOException {
822 if (cellCount < 7) {
823 return false;
824 }
825 cellCount++;
826 return true;
827 }
828
829 @Override
830 public String toString() {
831 return this.getClass().getSimpleName();
832 }
833
834 public static LimitKVsReturnFilter parseFrom(final byte [] pbBytes)
835 throws DeserializationException {
836 return new LimitKVsReturnFilter();
837 }
838 }
839 }