1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.junit.Assert.assertEquals;
21
22 import com.google.common.base.Preconditions;
23 import java.io.IOException;
24 import java.util.List;
25 import java.util.concurrent.ThreadLocalRandom;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.DoNotRetryIOException;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.MiniHBaseCluster;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.Waiter;
36 import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
37 import org.apache.hadoop.hbase.client.HBaseAdmin;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Table;
40 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
41 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
42 import org.apache.hadoop.hbase.regionserver.HRegionServer;
43 import org.apache.hadoop.hbase.regionserver.HStore;
44 import org.apache.hadoop.hbase.regionserver.Region;
45 import org.apache.hadoop.hbase.regionserver.Store;
46 import org.apache.hadoop.hbase.regionserver.StoreFile;
47 import org.apache.hadoop.hbase.testclassification.MediumTests;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.hbase.util.EnvironmentEdge;
50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51 import org.apache.hadoop.hbase.util.JVMClusterUtil;
52 import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
53 import org.junit.AfterClass;
54 import org.junit.Assert;
55 import org.junit.BeforeClass;
56 import org.junit.Rule;
57 import org.junit.Test;
58 import org.junit.experimental.categories.Category;
59 import org.junit.rules.ExpectedException;
60
61 @Category({ MediumTests.class })
62 public class TestFIFOCompactionPolicy {
63
64 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65
66 private final byte[] family = Bytes.toBytes("f");
67
68 private final byte[] qualifier = Bytes.toBytes("q");
69
70 @Rule
71 public ExpectedException error = ExpectedException.none();
72
73 private Store getStoreWithName(TableName tableName) {
74 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
75 List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
76 for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
77 HRegionServer hrs = rsts.get(i).getRegionServer();
78 for (Region region : hrs.getOnlineRegions(tableName)) {
79 return region.getStores().iterator().next();
80 }
81 }
82 return null;
83 }
84
85 private Store prepareData(TableName tableName) throws IOException {
86 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
87 HTableDescriptor desc = new HTableDescriptor(tableName);
88 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
89 FIFOCompactionPolicy.class.getName());
90 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91 DisabledRegionSplitPolicy.class.getName());
92 HColumnDescriptor colDesc = new HColumnDescriptor(family);
93 colDesc.setTimeToLive(1);
94 desc.addFamily(colDesc);
95
96 admin.createTable(desc);
97 Table table = TEST_UTIL.getConnection().getTable(tableName);
98 TimeOffsetEnvironmentEdge edge =
99 (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
100 for (int i = 0; i < 10; i++) {
101 for (int j = 0; j < 10; j++) {
102 byte[] value = new byte[128 * 1024];
103 ThreadLocalRandom.current().nextBytes(value);
104 table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
105 }
106 admin.flush(tableName);
107 edge.increment(1001);
108 }
109 return getStoreWithName(tableName);
110 }
111
112 @BeforeClass
113 public static void setEnvironmentEdge() throws Exception {
114 EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
115 EnvironmentEdgeManager.injectEdge(ee);
116 Configuration conf = TEST_UTIL.getConfiguration();
117 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
118
119
120
121
122 conf.setBoolean("hbase.store.delete.expired.storefile", false);
123 TEST_UTIL.startMiniCluster(1);
124 }
125
126 @AfterClass
127 public static void resetEnvironmentEdge() throws Exception {
128 TEST_UTIL.shutdownMiniCluster();
129 EnvironmentEdgeManager.reset();
130 }
131
132 @Test
133 public void testPurgeExpiredFiles() throws Exception {
134 TableName tableName = TableName.valueOf(getClass().getSimpleName());
135 final Store store = prepareData(tableName);
136 assertEquals(10, store.getStorefilesCount());
137 TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
138 TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
139
140 @Override
141 public boolean evaluate() throws Exception {
142 return store.getStorefilesCount() == 1;
143 }
144
145 @Override
146 public String explainFailure() throws Exception {
147 return "The store file count " + store.getStorefilesCount() + " is still greater than 1";
148 }
149 });
150 }
151
152 @Test
153 public void testSanityCheckTTL() throws IOException {
154 error.expect(DoNotRetryIOException.class);
155 error.expectMessage("Default TTL is not supported");
156 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL");
157 HTableDescriptor desc = new HTableDescriptor(tableName);
158 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
159 FIFOCompactionPolicy.class.getName());
160 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
161 DisabledRegionSplitPolicy.class.getName());
162 HColumnDescriptor colDesc = new HColumnDescriptor(family);
163 desc.addFamily(colDesc);
164 TEST_UTIL.getHBaseAdmin().createTable(desc);
165 }
166
167 @Test
168 public void testSanityCheckMinVersion() throws IOException {
169 error.expect(DoNotRetryIOException.class);
170 error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction");
171 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion");
172 HTableDescriptor desc = new HTableDescriptor(tableName);
173 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
174 FIFOCompactionPolicy.class.getName());
175 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
176 DisabledRegionSplitPolicy.class.getName());
177 HColumnDescriptor colDesc = new HColumnDescriptor(family);
178 colDesc.setTimeToLive(1);
179 colDesc.setMinVersions(1);
180 desc.addFamily(colDesc);
181 TEST_UTIL.getHBaseAdmin().createTable(desc);
182 }
183
184 @Test
185 public void testSanityCheckBlockingStoreFiles() throws IOException {
186 error.expect(DoNotRetryIOException.class);
187 error.expectMessage("blocking file count 'hbase.hstore.blockingStoreFiles'");
188 error.expectMessage("is below recommended minimum of 1000");
189 TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles");
190 HTableDescriptor desc = new HTableDescriptor(tableName);
191 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
192 FIFOCompactionPolicy.class.getName());
193 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
194 DisabledRegionSplitPolicy.class.getName());
195 desc.setConfiguration(HStore.BLOCKING_STOREFILES_KEY, "10");
196 HColumnDescriptor colDesc = new HColumnDescriptor(family);
197 colDesc.setTimeToLive(1);
198 desc.addFamily(colDesc);
199 TEST_UTIL.getHBaseAdmin().createTable(desc);
200 }
201
202
203
204
205 @Test
206 public void testFIFOCompactionPolicyExpiredEmptyHFiles() throws Exception {
207 TableName tableName = TableName.valueOf("testFIFOCompactionPolicyExpiredEmptyHFiles");
208 HColumnDescriptor colDesc = new HColumnDescriptor(family).setTimeToLive(1);
209 HTableDescriptor desc = new HTableDescriptor(tableName)
210 .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
211 FIFOCompactionPolicy.class.getName())
212 .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
213 DisabledRegionSplitPolicy.class.getName())
214 .addFamily(colDesc);
215 Table table = TEST_UTIL.createTable(desc, null);
216 long ts = System.currentTimeMillis() - 10 * 1000;
217 Put put =
218 new Put(Bytes.toBytes("row1")).addColumn(family, qualifier, ts, Bytes.toBytes("value0"));
219 table.put(put);
220 TEST_UTIL.getHBaseAdmin().flush(tableName);
221 put = new Put(Bytes.toBytes("row2")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
222 table.put(put);
223 TEST_UTIL.getHBaseAdmin().flush(tableName);
224 final Store store = Preconditions.checkNotNull(getStoreWithName(tableName));
225 Assert.assertEquals(2, store.getStorefilesCount());
226 TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
227 final int testWaitTimeoutMs = 20000;
228 TEST_UTIL.waitFor(testWaitTimeoutMs, new Waiter.Predicate<Exception>() {
229 @Override
230 public boolean evaluate() {
231 return store.getStorefilesCount() == 1;
232 }
233 });
234 StoreFile sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
235
236 Assert.assertEquals(0, sf.getReader().getEntries());
237 put = new Put(Bytes.toBytes("row3")).addColumn(family, qualifier, ts, Bytes.toBytes("value1"));
238 table.put(put);
239 TEST_UTIL.getHBaseAdmin().flush(tableName);
240 Assert.assertEquals(2, store.getStorefilesCount());
241 TEST_UTIL.getHBaseAdmin().majorCompact(tableName);
242 TEST_UTIL.waitFor(testWaitTimeoutMs, new Waiter.Predicate<Exception>() {
243 @Override
244 public boolean evaluate() {
245 return store.getStorefilesCount() == 1;
246 }
247 });
248 sf = Preconditions.checkNotNull(store.getStorefiles().iterator().next());
249 Assert.assertEquals(0, sf.getReader().getEntries());
250 }
251 }