1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.List;
31 import java.util.Random;
32 import java.util.concurrent.CountDownLatch;
33 import java.util.concurrent.ExecutorService;
34 import java.util.concurrent.Executors;
35 import java.util.concurrent.ThreadLocalRandom;
36 import java.util.concurrent.TimeUnit;
37 import static junit.framework.Assert.assertFalse;
38 import org.apache.commons.logging.Log;
39 import org.apache.commons.logging.LogFactory;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hadoop.hbase.Cell;
42 import org.apache.hadoop.hbase.CellUtil;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionLocation;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.client.coprocessor.Batch;
50 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
51 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
52 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
53 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
54 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
55 import org.apache.hadoop.hbase.Coprocessor;
56 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
57 import org.apache.hadoop.hbase.ipc.ServerRpcController;
58 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
59 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
60 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
61 import org.apache.hadoop.hbase.regionserver.HRegion;
62 import org.apache.hadoop.hbase.regionserver.HRegionServer;
63 import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
64 import org.apache.hadoop.hbase.regionserver.Region;
65 import org.apache.hadoop.hbase.testclassification.LargeTests;
66 import org.apache.hadoop.hbase.util.Bytes;
67 import org.apache.hadoop.hbase.util.Pair;
68 import org.junit.After;
69 import org.junit.AfterClass;
70 import org.junit.Assert;
71 import org.junit.Before;
72 import org.junit.BeforeClass;
73 import org.junit.Test;
74 import org.junit.experimental.categories.Category;
75
76 @Category(LargeTests.class)
77 public class TestFromClientSide3 {
78 private static final Log LOG = LogFactory.getLog(TestFromClientSide3.class);
79 private final static HBaseTestingUtility TEST_UTIL
80 = new HBaseTestingUtility();
81 private static byte[] FAMILY = Bytes.toBytes("testFamily");
82 private static Random random = new Random();
83 private static int SLAVES = 3;
84 private static final byte [] ROW = Bytes.toBytes("testRow");
85 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
86 private static final byte [] QUALIFIER = Bytes.toBytes("testQualifier");
87 private static final byte [] VALUE = Bytes.toBytes("testValue");
88 private static final byte[] COL_QUAL = Bytes.toBytes("f1");
89 private static final byte[] VAL_BYTES = Bytes.toBytes("v1");
90 private static final byte[] ROW_BYTES = Bytes.toBytes("r1");
91
92
93
94
95 @BeforeClass
96 public static void setUpBeforeClass() throws Exception {
97 TEST_UTIL.getConfiguration().setBoolean(
98 "hbase.online.schema.update.enable", true);
99 TEST_UTIL.startMiniCluster(SLAVES);
100 }
101
102
103
104
105 @AfterClass
106 public static void tearDownAfterClass() throws Exception {
107 TEST_UTIL.shutdownMiniCluster();
108 }
109
110
111
112
113 @Before
114 public void setUp() throws Exception {
115
116 }
117
118
119
120
121 @After
122 public void tearDown() throws Exception {
123 for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
124 LOG.info("Tear down, remove table=" + htd.getTableName());
125 TEST_UTIL.deleteTable(htd.getTableName());
126 }
127 }
128
129 private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
130 throws Exception {
131 Put put = new Put(row);
132 for (int i = 0; i < nPuts; i++) {
133 byte[] qualifier = Bytes.toBytes(random.nextInt());
134 byte[] value = Bytes.toBytes(random.nextInt());
135 put.add(family, qualifier, value);
136 }
137 table.put(put);
138 }
139
140 private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
141 byte[] row, byte[] family, int nFlushes, int nPuts)
142 throws Exception {
143
144
145 HRegionLocation loc = table.getRegionLocation(row, true);
146 AdminProtos.AdminService.BlockingInterface server =
147 admin.getConnection().getAdmin(loc.getServerName());
148 byte[] regName = loc.getRegionInfo().getRegionName();
149
150 for (int i = 0; i < nFlushes; i++) {
151 randomCFPuts(table, row, family, nPuts);
152 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
153 int sfCount = sf.size();
154
155
156 admin.flush(table.getTableName());
157
158
159 while (ProtobufUtil.getStoreFiles(
160 server, regName, FAMILY).size() == sfCount) {
161 Thread.sleep(40);
162 }
163 }
164 }
165
166 private static List<Cell> toList(ResultScanner scanner) {
167 try {
168 List<Cell> cells = new ArrayList<>();
169 for (Result r : scanner) {
170 cells.addAll(r.listCells());
171 }
172 return cells;
173 } finally {
174 scanner.close();
175 }
176 }
177
178 @Test
179 public void testScanAfterDeletingSpecifiedRow() throws IOException {
180 TableName tableName = TableName.valueOf("testScanAfterDeletingSpecifiedRow");
181 HTableDescriptor desc = new HTableDescriptor(tableName);
182 desc.addFamily(new HColumnDescriptor(FAMILY));
183 TEST_UTIL.getHBaseAdmin().createTable(desc);
184 byte[] row = Bytes.toBytes("SpecifiedRow");
185 byte[] value0 = Bytes.toBytes("value_0");
186 byte[] value1 = Bytes.toBytes("value_1");
187 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
188 Put put = new Put(row);
189 put.addColumn(FAMILY, QUALIFIER, VALUE);
190 t.put(put);
191 Delete d = new Delete(row);
192 t.delete(d);
193 put = new Put(row);
194 put.addColumn(FAMILY, null, value0);
195 t.put(put);
196 put = new Put(row);
197 put.addColumn(FAMILY, null, value1);
198 t.put(put);
199 List<Cell> cells = toList(t.getScanner(new Scan()));
200 assertEquals(1, cells.size());
201 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
202
203 cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
204 assertEquals(1, cells.size());
205 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
206
207 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
208 assertEquals(0, cells.size());
209
210 TEST_UTIL.getHBaseAdmin().flush(tableName);
211 cells = toList(t.getScanner(new Scan()));
212 assertEquals(1, cells.size());
213 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
214
215 cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
216 assertEquals(1, cells.size());
217 assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
218
219 cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
220 assertEquals(0, cells.size());
221 }
222 }
223
224 @Test
225 public void testScanAfterDeletingSpecifiedRowV2() throws IOException {
226 TableName tableName = TableName.valueOf("testScanAfterDeletingSpecifiedRowV2");
227 HTableDescriptor desc = new HTableDescriptor(tableName);
228 desc.addFamily(new HColumnDescriptor(FAMILY));
229 TEST_UTIL.getHBaseAdmin().createTable(desc);
230 byte[] row = Bytes.toBytes("SpecifiedRow");
231 byte[] qual0 = Bytes.toBytes("qual0");
232 byte[] qual1 = Bytes.toBytes("qual1");
233 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
234 Delete d = new Delete(row);
235 t.delete(d);
236
237 Put put = new Put(row);
238 put.addColumn(FAMILY, null, VALUE);
239 t.put(put);
240
241 put = new Put(row);
242 put.addColumn(FAMILY, qual1, qual1);
243 t.put(put);
244
245 put = new Put(row);
246 put.addColumn(FAMILY, qual0, qual0);
247 t.put(put);
248
249 Result r = t.get(new Get(row));
250 assertEquals(3, r.size());
251 assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0])));
252 assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1])));
253 assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2])));
254
255 TEST_UTIL.getHBaseAdmin().flush(tableName);
256 r = t.get(new Get(row));
257 assertEquals(3, r.size());
258 assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0])));
259 assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1])));
260 assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2])));
261 }
262 }
263
264
265 @Test(timeout = 60000)
266 public void testAdvancedConfigOverride() throws Exception {
267
268
269
270
271
272
273
274
275
276 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
277
278 String tableName = "testAdvancedConfigOverride";
279 TableName TABLE = TableName.valueOf(tableName);
280 HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
281 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
282 ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
283
284
285 byte[] row = Bytes.toBytes(random.nextInt());
286 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
287
288
289 HRegionLocation loc = hTable.getRegionLocation(row, true);
290 byte[] regionName = loc.getRegionInfo().getRegionName();
291 AdminProtos.AdminService.BlockingInterface server =
292 connection.getAdmin(loc.getServerName());
293 assertTrue(ProtobufUtil.getStoreFiles(
294 server, regionName, FAMILY).size() > 1);
295
296
297 admin.compact(TABLE.getName());
298
299
300 for (int i = 0; i < 10 * 1000 / 40; ++i) {
301
302 loc = hTable.getRegionLocation(row, true);
303 if (!loc.getRegionInfo().isOffline()) {
304 regionName = loc.getRegionInfo().getRegionName();
305 server = connection.getAdmin(loc.getServerName());
306 if (ProtobufUtil.getStoreFiles(
307 server, regionName, FAMILY).size() <= 1) {
308 break;
309 }
310 }
311 Thread.sleep(40);
312 }
313
314 assertTrue(ProtobufUtil.getStoreFiles(
315 server, regionName, FAMILY).size() <= 1);
316
317
318 LOG.info("hbase.hstore.compaction.min should now be 5");
319 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
320 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
321 admin.modifyTable(TABLE, htd);
322 Pair<Integer, Integer> st;
323 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
324 LOG.debug(st.getFirst() + " regions left to update");
325 Thread.sleep(40);
326 }
327 LOG.info("alter status finished");
328
329
330 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
331
332
333 admin.compact(TABLE.getName());
334
335
336 Thread.sleep(10 * 1000);
337 loc = hTable.getRegionLocation(row, true);
338 regionName = loc.getRegionInfo().getRegionName();
339 server = connection.getAdmin(loc.getServerName());
340 int sfCount = ProtobufUtil.getStoreFiles(
341 server, regionName, FAMILY).size();
342 assertTrue(sfCount > 1);
343
344
345 LOG.info("hbase.hstore.compaction.min should now be 2");
346 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
347 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
348 htd.modifyFamily(hcd);
349 admin.modifyTable(TABLE, htd);
350 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
351 LOG.debug(st.getFirst() + " regions left to update");
352 Thread.sleep(40);
353 }
354 LOG.info("alter status finished");
355
356
357 admin.compact(TABLE.getName());
358
359
360 for (int i = 0; i < 10 * 1000 / 40; ++i) {
361 loc = hTable.getRegionLocation(row, true);
362 regionName = loc.getRegionInfo().getRegionName();
363 try {
364 server = connection.getAdmin(loc.getServerName());
365 if (ProtobufUtil.getStoreFiles(
366 server, regionName, FAMILY).size() < sfCount) {
367 break;
368 }
369 } catch (Exception e) {
370 LOG.debug("Waiting for region to come online: " + Bytes.toString(regionName));
371 }
372 Thread.sleep(40);
373 }
374
375 assertTrue(ProtobufUtil.getStoreFiles(
376 server, regionName, FAMILY).size() < sfCount);
377
378
379 LOG.info("Removing CF config value");
380 LOG.info("hbase.hstore.compaction.min should now be 5");
381 hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
382 hcd.setValue("hbase.hstore.compaction.min", null);
383 htd.modifyFamily(hcd);
384 admin.modifyTable(TABLE, htd);
385 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
386 LOG.debug(st.getFirst() + " regions left to update");
387 Thread.sleep(40);
388 }
389 LOG.info("alter status finished");
390 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
391 "hbase.hstore.compaction.min"));
392 }
393
394 @Test
395 public void testHTableBatchWithEmptyPut() throws Exception {
396 Table table = TEST_UTIL.createTable(
397 Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
398 try {
399 List actions = (List) new ArrayList();
400 Object[] results = new Object[2];
401
402 Put put1 = new Put(ROW);
403 actions.add(put1);
404
405 Put put2 = new Put(ANOTHERROW);
406 put2.add(FAMILY, QUALIFIER, VALUE);
407 actions.add(put2);
408
409 table.batch(actions, results);
410 fail("Empty Put should have failed the batch call");
411 } catch (IllegalArgumentException iae) {
412
413 } finally {
414 table.close();
415 }
416 }
417
418
419
420 @Test
421 public void testHTableWithLargeBatch() throws Exception {
422 Table table = TEST_UTIL.createTable(TableName.valueOf("testHTableWithLargeBatch"),
423 new byte[][] { FAMILY });
424 int sixtyFourK = 64 * 1024;
425 try {
426 List actions = new ArrayList();
427 Object[] results = new Object[(sixtyFourK + 1) * 2];
428
429 for (int i = 0; i < sixtyFourK + 1; i ++) {
430 Put put1 = new Put(ROW);
431 put1.addColumn(FAMILY, QUALIFIER, VALUE);
432 actions.add(put1);
433
434 Put put2 = new Put(ANOTHERROW);
435 put2.addColumn(FAMILY, QUALIFIER, VALUE);
436 actions.add(put2);
437 }
438
439 table.batch(actions, results);
440 } finally {
441 table.close();
442 }
443 }
444
445 @Test
446 public void testBatchWithRowMutation() throws Exception {
447 LOG.info("Starting testBatchWithRowMutation");
448 final TableName TABLENAME = TableName.valueOf("testBatchWithRowMutation");
449 try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) {
450 byte [][] QUALIFIERS = new byte [][] {
451 Bytes.toBytes("a"), Bytes.toBytes("b")
452 };
453 RowMutations arm = new RowMutations(ROW);
454 Put p = new Put(ROW);
455 p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
456 arm.add(p);
457 Object[] batchResult = new Object[1];
458 t.batch(Arrays.asList(arm), batchResult);
459
460 Get g = new Get(ROW);
461 Result r = t.get(g);
462 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
463
464 arm = new RowMutations(ROW);
465 p = new Put(ROW);
466 p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
467 arm.add(p);
468 Delete d = new Delete(ROW);
469 d.addColumns(FAMILY, QUALIFIERS[0]);
470 arm.add(d);
471 t.batch(Arrays.asList(arm), batchResult);
472 r = t.get(g);
473 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
474 assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
475
476
477 try {
478 arm = new RowMutations(ROW);
479 p = new Put(ROW);
480 p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
481 arm.add(p);
482 t.batch(Arrays.asList(arm), batchResult);
483 fail("Expected RetriesExhaustedWithDetailsException with NoSuchColumnFamilyException");
484 } catch (RetriesExhaustedWithDetailsException e) {
485 String msg = e.getMessage();
486 assertTrue(msg.contains("NoSuchColumnFamilyException"));
487 }
488 }
489 }
490
491 @Test
492 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
493
494
495
496 Table table = TEST_UTIL.createTable(
497 Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
498
499 Put put = new Put(ROW);
500 put.add(FAMILY, QUALIFIER, VALUE);
501
502 Get get = new Get(ROW);
503
504 boolean exist = table.exists(get);
505 assertEquals(exist, false);
506
507 table.put(put);
508
509 exist = table.exists(get);
510 assertEquals(exist, true);
511 }
512
513 @Test
514 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
515
516 HTable table = TEST_UTIL.createTable(
517 Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
518
519 Put put = new Put(ROW);
520 put.add(FAMILY, QUALIFIER, VALUE);
521 table.put(put);
522
523 List<Get> gets = new ArrayList<Get>();
524 gets.add(new Get(ROW));
525 gets.add(new Get(ANOTHERROW));
526
527 Boolean[] results = table.exists(gets);
528 assertTrue(results[0]);
529 assertFalse(results[1]);
530 }
531
532 @Test
533 public void testHTableExistsBeforeGet() throws Exception {
534 Table table = TEST_UTIL.createTable(
535 Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
536 try {
537 Put put = new Put(ROW);
538 put.add(FAMILY, QUALIFIER, VALUE);
539 table.put(put);
540
541 Get get = new Get(ROW);
542
543 boolean exist = table.exists(get);
544 assertEquals(true, exist);
545
546 Result result = table.get(get);
547 assertEquals(false, result.isEmpty());
548 assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
549 } finally {
550 table.close();
551 }
552 }
553
554 @Test
555 public void testHTableExistsAllBeforeGet() throws Exception {
556 final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
557 Table table = TEST_UTIL.createTable(
558 Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
559 try {
560 Put put = new Put(ROW);
561 put.add(FAMILY, QUALIFIER, VALUE);
562 table.put(put);
563 put = new Put(ROW2);
564 put.add(FAMILY, QUALIFIER, VALUE);
565 table.put(put);
566
567 Get get = new Get(ROW);
568 Get get2 = new Get(ROW2);
569 ArrayList<Get> getList = new ArrayList(2);
570 getList.add(get);
571 getList.add(get2);
572
573 boolean[] exists = table.existsAll(getList);
574 assertEquals(true, exists[0]);
575 assertEquals(true, exists[1]);
576
577 Result[] result = table.get(getList);
578 assertEquals(false, result[0].isEmpty());
579 assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
580 assertEquals(false, result[1].isEmpty());
581 assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
582 } finally {
583 table.close();
584 }
585 }
586
587 @Test
588 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
589
590 Table table = TEST_UTIL.createTable(
591 TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
592 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
593 Put put = new Put(ROW);
594 put.add(FAMILY, QUALIFIER, VALUE);
595
596 Get get = new Get(ROW);
597
598 boolean exist = table.exists(get);
599 assertEquals(exist, false);
600
601 table.put(put);
602
603 exist = table.exists(get);
604 assertEquals(exist, true);
605 }
606
607 @Test
608 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
609 HTable table = TEST_UTIL.createTable(
610 TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
611 new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
612 Put put = new Put(ROW);
613 put.add(FAMILY, QUALIFIER, VALUE);
614 table.put (put);
615
616 List<Get> gets = new ArrayList<Get>();
617 gets.add(new Get(ANOTHERROW));
618 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
619 gets.add(new Get(ROW));
620 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
621
622 LOG.info("Calling exists");
623 Boolean[] results = table.exists(gets);
624 assertEquals(results[0], false);
625 assertEquals(results[1], false);
626 assertEquals(results[2], true);
627 assertEquals(results[3], false);
628
629
630 put = new Put(new byte[] { 0x00 });
631 put.add(FAMILY, QUALIFIER, VALUE);
632 table.put(put);
633
634 gets = new ArrayList<Get>();
635 gets.add(new Get(new byte[] { 0x00 }));
636 gets.add(new Get(new byte[] { 0x00, 0x00 }));
637 results = table.exists(gets);
638 assertEquals(results[0], true);
639 assertEquals(results[1], false);
640
641
642 put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
643 put.add(FAMILY, QUALIFIER, VALUE);
644 table.put(put);
645
646 gets = new ArrayList<Get>();
647 gets.add(new Get(new byte[] { (byte) 0xff }));
648 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
649 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
650 results = table.exists(gets);
651 assertEquals(results[0], false);
652 assertEquals(results[1], true);
653 assertEquals(results[2], false);
654 }
655
656 @Test
657 public void testGetEmptyRow() throws Exception {
658
659 Admin admin = TEST_UTIL.getHBaseAdmin();
660 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
661 desc.addFamily(new HColumnDescriptor(FAMILY));
662 admin.createTable(desc);
663 Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
664
665 Put put = new Put(ROW_BYTES);
666 put.add(FAMILY, COL_QUAL, VAL_BYTES);
667 table.put(put);
668
669
670 Result res = null;
671 try {
672 res = table.get(new Get(new byte[0]));
673 fail();
674 } catch (IllegalArgumentException e) {
675
676 }
677 assertTrue(res == null);
678 res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
679 assertTrue(res.isEmpty() == true);
680 res = table.get(new Get(ROW_BYTES));
681 assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
682 table.close();
683 }
684
685 @Test(timeout = 30000)
686 public void testMultiRowMutations() throws Exception, Throwable {
687 final TableName tableName = TableName.valueOf("testMultiRowMutations");
688 HTableDescriptor desc = new HTableDescriptor(tableName);
689 desc.addCoprocessor(MultiRowMutationEndpoint.class.getName());
690 desc.addCoprocessor(WatiingForMultiMutationsObserver.class.getName());
691 desc.setConfiguration("hbase.rowlock.wait.duration", String.valueOf(5000));
692 desc.addFamily(new HColumnDescriptor(FAMILY));
693 TEST_UTIL.getHBaseAdmin().createTable(desc);
694
695 Configuration copy = new Configuration(TEST_UTIL.getConfiguration());
696 copy.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
697 try (Connection con = ConnectionFactory.createConnection(copy)) {
698 final byte[] row = Bytes.toBytes("ROW-0");
699 final byte[] rowLocked= Bytes.toBytes("ROW-1");
700 final byte[] value0 = Bytes.toBytes("VALUE-0");
701 final byte[] value1 = Bytes.toBytes("VALUE-1");
702 final byte[] value2 = Bytes.toBytes("VALUE-2");
703 assertNoLocks(tableName);
704 ExecutorService putService = Executors.newSingleThreadExecutor();
705 putService.execute(new Runnable() {
706 @Override
707 public void run() {
708 try (Table table = con.getTable(tableName)) {
709 Put put0 = new Put(rowLocked);
710 put0.addColumn(FAMILY, QUALIFIER, value0);
711
712 table.put(put0);
713 } catch (IOException ex) {
714 throw new RuntimeException(ex);
715 }
716 }
717 });
718 ExecutorService cpService = Executors.newSingleThreadExecutor();
719 cpService.execute(new Runnable() {
720 @Override
721 public void run() {
722 boolean threw;
723 Put put1 = new Put(row);
724 Put put2 = new Put(rowLocked);
725 put1.addColumn(FAMILY, QUALIFIER, value1);
726 put2.addColumn(FAMILY, QUALIFIER, value2);
727 try (Table table = con.getTable(tableName)) {
728 final MultiRowMutationProtos.MutateRowsRequest request
729 = MultiRowMutationProtos.MutateRowsRequest.newBuilder()
730 .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation(
731 org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, put1))
732 .addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation(
733 org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, put2))
734 .build();
735 table.coprocessorService(MultiRowMutationProtos.MultiRowMutationService.class, ROW, ROW,
736 new Batch.Call<MultiRowMutationProtos.MultiRowMutationService, MultiRowMutationProtos.MutateRowsResponse>() {
737 public MultiRowMutationProtos.MutateRowsResponse call(MultiRowMutationProtos.MultiRowMutationService instance) throws IOException {
738 ServerRpcController controller = new ServerRpcController();
739 BlockingRpcCallback<MultiRowMutationProtos.MutateRowsResponse> rpcCallback = new BlockingRpcCallback<>();
740 instance.mutateRows(controller, request, rpcCallback);
741 return rpcCallback.get();
742 }
743 });
744 threw = false;
745 } catch (Throwable ex) {
746 threw = true;
747 }
748 if (!threw) {
749
750 fail("This cp should fail because the target lock is blocked by previous put");
751 }
752 }
753 });
754 cpService.shutdown();
755 cpService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
756 WatiingForMultiMutationsObserver observer = find(tableName, WatiingForMultiMutationsObserver.class);
757 observer.latch.countDown();
758 putService.shutdown();
759 putService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
760 try (Table table = con.getTable(tableName)) {
761 Get g0 = new Get(row);
762 Get g1 = new Get(rowLocked);
763 Result r0 = table.get(g0);
764 Result r1 = table.get(g1);
765 assertTrue(r0.isEmpty());
766 assertFalse(r1.isEmpty());
767 assertTrue(Bytes.equals(r1.getValue(FAMILY, QUALIFIER), value0));
768 }
769 assertNoLocks(tableName);
770 }
771 }
772
773 private static void assertNoLocks(final TableName tableName) throws IOException, InterruptedException {
774 HRegion region = (HRegion) find(tableName);
775 assertEquals(0, region.getLockedRows().size());
776 }
777 private static Region find(final TableName tableName)
778 throws IOException, InterruptedException {
779 HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(tableName);
780 List<Region> regions = rs.getOnlineRegions(tableName);
781 assertEquals(1, regions.size());
782 return regions.get(0);
783 }
784
785 private static <T extends RegionObserver> T find(final TableName tableName,
786 Class<T> clz) throws IOException, InterruptedException {
787 Region region = find(tableName);
788 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(clz.getName());
789 assertTrue("The cp instance should be " + clz.getName()
790 + ", current instance is " + cp.getClass().getName(), clz.isInstance(cp));
791 return clz.cast(cp);
792 }
793
794 public static class WatiingForMultiMutationsObserver extends BaseRegionObserver {
795 final CountDownLatch latch = new CountDownLatch(1);
796 @Override
797 public void preBatchMutate(final ObserverContext<RegionCoprocessorEnvironment> c,
798 final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
799 try {
800 latch.await();
801 } catch (InterruptedException ex) {
802 throw new IOException(ex);
803 }
804 }
805 }
806
807 private static byte[] generateHugeValue(int size) {
808 Random rand = ThreadLocalRandom.current();
809 byte[] value = new byte[size];
810 for (int i = 0; i < value.length; i++) {
811 value[i] = (byte) rand.nextInt(256);
812 }
813 return value;
814 }
815
816 @Test
817 public void testScanWithBatchSizeReturnIncompleteCells() throws IOException {
818 TableName tableName = TableName.valueOf("testScanWithBatchSizeReturnIncompleteCells");
819 HTableDescriptor hd = new HTableDescriptor(tableName);
820 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY).setMaxVersions(3);
821 hd.addFamily(hcd);
822
823 Table table = TEST_UTIL.createTable(hd, null);
824
825 Put put = new Put(ROW);
826 put.addColumn(FAMILY, Bytes.toBytes(0), generateHugeValue(3 * 1024 * 1024));
827 table.put(put);
828
829 put = new Put(ROW);
830 put.addColumn(FAMILY, Bytes.toBytes(1), generateHugeValue(4 * 1024 * 1024));
831 table.put(put);
832
833 for (int i = 2; i < 5; i++) {
834 for (int version = 0; version < 2; version++) {
835 put = new Put(ROW);
836 put.addColumn(FAMILY, Bytes.toBytes(i), generateHugeValue(1024));
837 table.put(put);
838 }
839 }
840
841 Scan scan = new Scan();
842 scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(3)
843 .setMaxResultSize(4 * 1024 * 1024);
844 Result result;
845 try (ResultScanner scanner = table.getScanner(scan)) {
846 List<Result> list = new ArrayList<>();
847
848
849
850
851
852
853
854 while ((result = scanner.next()) != null) {
855 list.add(result);
856 }
857
858 Assert.assertEquals(2, list.size());
859 Assert.assertEquals(3, list.get(0).size());
860 Assert.assertEquals(2, list.get(1).size());
861 }
862
863 scan = new Scan();
864 scan.withStartRow(ROW).withStopRow(ROW, true).addFamily(FAMILY).setBatch(2)
865 .setMaxResultSize(4 * 1024 * 1024);
866 try (ResultScanner scanner = table.getScanner(scan)) {
867 List<Result> list = new ArrayList<>();
868 while ((result = scanner.next()) != null) {
869 list.add(result);
870 }
871 Assert.assertEquals(3, list.size());
872 Assert.assertEquals(2, list.get(0).size());
873 Assert.assertEquals(2, list.get(1).size());
874 Assert.assertEquals(1, list.get(2).size());
875 }
876 }
877 }