1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.util.Collection;
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.fs.FileSystem;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.*;
27 import org.apache.hadoop.hbase.client.*;
28 import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;
29 import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
30 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
31 import org.apache.hadoop.hbase.testclassification.MediumTests;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.hbase.util.FSUtils;
34 import org.junit.After;
35 import org.junit.AfterClass;
36 import org.junit.Assert;
37 import org.junit.BeforeClass;
38 import org.junit.Test;
39
40 import org.junit.experimental.categories.Category;
41
42 import static org.junit.Assert.assertEquals;
43 import static org.junit.Assert.assertTrue;
44
45 @Category(MediumTests.class)
46 public class TestCompactSplitThread {
47 private static final Log LOG = LogFactory.getLog(TestCompactSplitThread.class);
48 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49 private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
50 private final byte[] family = Bytes.toBytes("f");
51 private static final int NUM_RS = 1;
52 private static final int blockingStoreFiles = 3;
53 private static Path rootDir;
54 private static FileSystem fs;
55
56
57
58
59
60
61 @BeforeClass
62 public static void setupCluster() throws Exception {
63 setupConf(TEST_UTIL.getConfiguration());
64 TEST_UTIL.startMiniCluster(NUM_RS);
65 fs = TEST_UTIL.getDFSCluster().getFileSystem();
66 rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
67
68 }
69
70 private static void setupConf(Configuration conf) {
71
72 conf.setInt("hbase.regionsever.info.port", -1);
73
74
75 conf.setInt("hbase.hstore.compaction.min", 2);
76 conf.setInt("hbase.hstore.compactionThreshold", 5);
77
78 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
79
80
81 conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles);
82
83 conf.setInt(CompactSplitThread.LARGE_COMPACTION_THREADS, 3);
84 conf.setInt(CompactSplitThread.SMALL_COMPACTION_THREADS, 4);
85 conf.setInt(CompactSplitThread.SPLIT_THREADS, 5);
86 conf.setInt(CompactSplitThread.MERGE_THREADS, 6);
87 }
88
89 @After
90 public void tearDown() throws Exception {
91 TEST_UTIL.deleteTable(tableName);
92 }
93
94 @AfterClass
95 public static void cleanupTest() throws Exception {
96 try {
97 TEST_UTIL.shutdownMiniCluster();
98 } catch (Exception e) {
99
100 }
101 }
102
103 @Test
104 public void testThreadPoolSizeTuning() throws Exception {
105 Configuration conf = TEST_UTIL.getConfiguration();
106 Connection conn = ConnectionFactory.createConnection(conf);
107 try {
108 HTableDescriptor htd = new HTableDescriptor(tableName);
109 htd.addFamily(new HColumnDescriptor(family));
110 htd.setCompactionEnabled(false);
111 TEST_UTIL.getHBaseAdmin().createTable(htd);
112 TEST_UTIL.waitTableAvailable(tableName);
113 HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName);
114
115
116 assertEquals(3, regionServer.compactSplitThread.getLargeCompactionThreadNum());
117 assertEquals(4, regionServer.compactSplitThread.getSmallCompactionThreadNum());
118 assertEquals(5, regionServer.compactSplitThread.getSplitThreadNum());
119 assertEquals(6, regionServer.compactSplitThread.getMergeThreadNum());
120
121
122 conf.setInt(CompactSplitThread.LARGE_COMPACTION_THREADS, 4);
123 conf.setInt(CompactSplitThread.SMALL_COMPACTION_THREADS, 5);
124 conf.setInt(CompactSplitThread.SPLIT_THREADS, 6);
125 conf.setInt(CompactSplitThread.MERGE_THREADS, 7);
126 try {
127 regionServer.compactSplitThread.onConfigurationChange(conf);
128 } catch (IllegalArgumentException iae) {
129 Assert.fail("Update bigger configuration failed!");
130 }
131
132
133 assertEquals(4, regionServer.compactSplitThread.getLargeCompactionThreadNum());
134 assertEquals(5, regionServer.compactSplitThread.getSmallCompactionThreadNum());
135 assertEquals(6, regionServer.compactSplitThread.getSplitThreadNum());
136 assertEquals(7, regionServer.compactSplitThread.getMergeThreadNum());
137
138
139 conf.setInt(CompactSplitThread.LARGE_COMPACTION_THREADS, 2);
140 conf.setInt(CompactSplitThread.SMALL_COMPACTION_THREADS, 3);
141 conf.setInt(CompactSplitThread.SPLIT_THREADS, 4);
142 conf.setInt(CompactSplitThread.MERGE_THREADS, 5);
143 try {
144 regionServer.compactSplitThread.onConfigurationChange(conf);
145 } catch (IllegalArgumentException iae) {
146 Assert.fail("Update smaller configuration failed!");
147 }
148
149
150 assertEquals(2, regionServer.compactSplitThread.getLargeCompactionThreadNum());
151 assertEquals(3, regionServer.compactSplitThread.getSmallCompactionThreadNum());
152 assertEquals(4, regionServer.compactSplitThread.getSplitThreadNum());
153 assertEquals(5, regionServer.compactSplitThread.getMergeThreadNum());
154 } finally {
155 conn.close();
156 }
157 }
158
159 @Test(timeout = 60000)
160 public void testFlushWithTableCompactionDisabled() throws Exception {
161 Admin admin = TEST_UTIL.getHBaseAdmin();
162
163 HTableDescriptor htd = new HTableDescriptor(tableName);
164 htd.setCompactionEnabled(false);
165 TEST_UTIL.createTable(htd, new byte[][] { family }, null);
166
167
168 for (int i = 0; i < blockingStoreFiles + 1; i ++) {
169 TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(tableName), family);
170 TEST_UTIL.flush(tableName);
171 }
172
173
174 Path tableDir = FSUtils.getTableDir(rootDir, tableName);
175 Collection<String> hfiles = SnapshotTestingUtils.listHFileNames(fs, tableDir);
176 assert(hfiles.size() > blockingStoreFiles + 1);
177 }
178 }