1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.Iterator;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HBaseTestCase;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.client.Durability;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.testclassification.SmallTests;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.junit.experimental.categories.Category;
43
44 @Category(SmallTests.class)
45 public class TestWideScanner extends HBaseTestCase {
46 private static final Log LOG = LogFactory.getLog(TestWideScanner.class);
47
48 static final byte[] A = Bytes.toBytes("A");
49 static final byte[] B = Bytes.toBytes("B");
50 static final byte[] C = Bytes.toBytes("C");
51 static byte[][] COLUMNS = { A, B, C };
52 static final Random rng = new Random();
53 static final HTableDescriptor TESTTABLEDESC =
54 new HTableDescriptor(TableName.valueOf("testwidescan"));
55 static {
56 for (byte[] cfName : new byte[][] { A, B, C }) {
57 TESTTABLEDESC.addFamily(new HColumnDescriptor(cfName)
58
59 .setMaxVersions(100)
60 .setBlocksize(8 * 1024)
61 );
62 }
63 }
64
65
66 HRegion r;
67
68 private int addWideContent(HRegion region) throws IOException {
69 int count = 0;
70 for (char c = 'a'; c <= 'c'; c++) {
71 byte[] row = Bytes.toBytes("ab" + c);
72 int i, j;
73 long ts = System.currentTimeMillis();
74 for (i = 0; i < 100; i++) {
75 byte[] b = Bytes.toBytes(String.format("%10d", i));
76 for (j = 0; j < 100; j++) {
77 Put put = new Put(row);
78 put.setDurability(Durability.SKIP_WAL);
79 put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b);
80 region.put(put);
81 count++;
82 }
83 }
84 }
85 return count;
86 }
87
88 public void testWideScanBatching() throws IOException {
89 final int batch = 256;
90 try {
91 this.r = createNewHRegion(TESTTABLEDESC, null, null);
92 int inserted = addWideContent(this.r);
93 List<Cell> results = new ArrayList<Cell>();
94 Scan scan = new Scan();
95 scan.addFamily(A);
96 scan.addFamily(B);
97 scan.addFamily(C);
98 scan.setMaxVersions(100);
99 scan.setBatch(batch);
100 InternalScanner s = r.getScanner(scan);
101 int total = 0;
102 int i = 0;
103 boolean more;
104 do {
105 more = s.next(results);
106 i++;
107 LOG.info("iteration #" + i + ", results.size=" + results.size());
108
109
110 assertTrue(results.size() <= batch);
111
112 total += results.size();
113
114 if (results.size() > 0) {
115
116 byte[] row = CellUtil.cloneRow(results.get(0));
117 for (Cell kv: results) {
118 assertTrue(Bytes.equals(row, CellUtil.cloneRow(kv)));
119 }
120 }
121
122 results.clear();
123
124
125 Iterator<KeyValueScanner> scanners =
126 ((HRegion.RegionScannerImpl)s).storeHeap.getHeap().iterator();
127 while (scanners.hasNext()) {
128 StoreScanner ss = (StoreScanner)scanners.next();
129 ss.updateReaders(Collections.EMPTY_LIST, Collections.EMPTY_LIST);
130 }
131 } while (more);
132
133
134 LOG.info("inserted " + inserted + ", scanned " + total);
135 assertEquals(total, inserted);
136
137 s.close();
138 } finally {
139 HRegion.closeHRegion(this.r);
140 }
141 }
142
143 }
144