View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.DoNotRetryIOException;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.ProcedureInfo;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
36  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
37  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
38  import org.apache.hadoop.hbase.testclassification.MediumTests;
39  import org.junit.After;
40  import org.junit.AfterClass;
41  import org.junit.Before;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  @Category(MediumTests.class)
47  public class TestModifyTableProcedure {
48    private static final Log LOG = LogFactory.getLog(TestModifyTableProcedure.class);
49  
50    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
51  
52    private static void setupConf(Configuration conf) {
53      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
54    }
55  
56    @BeforeClass
57    public static void setupCluster() throws Exception {
58      setupConf(UTIL.getConfiguration());
59      UTIL.startMiniCluster(1);
60    }
61  
62    @AfterClass
63    public static void cleanupTest() throws Exception {
64      try {
65        UTIL.shutdownMiniCluster();
66      } catch (Exception e) {
67        LOG.warn("failure shutting down cluster", e);
68      }
69    }
70  
71    @Before
72    public void setup() throws Exception {
73      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
74    }
75  
76    @After
77    public void tearDown() throws Exception {
78      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
79      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
80        LOG.info("Tear down, remove table=" + htd.getTableName());
81        UTIL.deleteTable(htd.getTableName());
82      }
83    }
84  
85    @Test(timeout=60000)
86    public void testModifyTable() throws Exception {
87      final TableName tableName = TableName.valueOf("testModifyTable");
88      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
89  
90      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
91      UTIL.getHBaseAdmin().disableTable(tableName);
92  
93      // Modify the table descriptor
94      HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
95  
96      // Test 1: Modify 1 property
97      long newMaxFileSize = htd.getMaxFileSize() * 2;
98      htd.setMaxFileSize(newMaxFileSize);
99      htd.setRegionReplication(3);
100 
101     long procId1 = ProcedureTestingUtility.submitAndWait(
102         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
103     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
104 
105     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
106     assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
107 
108     // Test 2: Modify multiple properties
109     boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
110     long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
111     htd.setReadOnly(newReadOnlyOption);
112     htd.setMemStoreFlushSize(newMemStoreFlushSize);
113 
114     long procId2 = ProcedureTestingUtility.submitAndWait(
115         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
116     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
117 
118     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
119     assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
120     assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
121   }
122 
123   @Test(timeout = 60000)
124   public void testModifyTableAddCF() throws Exception {
125     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
126     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
127 
128     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
129     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
130     assertEquals(1, currentHtd.getFamiliesKeys().size());
131 
132     // Test 1: Modify the table descriptor online
133     String cf2 = "cf2";
134     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
135     htd.addFamily(new HColumnDescriptor(cf2));
136 
137     long procId = ProcedureTestingUtility.submitAndWait(
138         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
139     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
140 
141     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
142     assertEquals(2, currentHtd.getFamiliesKeys().size());
143     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
144 
145     // Test 2: Modify the table descriptor offline
146     UTIL.getHBaseAdmin().disableTable(tableName);
147     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
148     String cf3 = "cf3";
149     HTableDescriptor htd2 =
150         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
151     htd2.addFamily(new HColumnDescriptor(cf3));
152 
153     long procId2 =
154         ProcedureTestingUtility.submitAndWait(procExec,
155           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
156     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
157 
158     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
159     assertTrue(currentHtd.hasFamily(cf3.getBytes()));
160     assertEquals(3, currentHtd.getFamiliesKeys().size());
161   }
162 
163   @Test(timeout = 60000)
164   public void testModifyTableDeleteCF() throws Exception {
165     final TableName tableName = TableName.valueOf("testModifyTableDeleteCF");
166     final String cf1 = "cf1";
167     final String cf2 = "cf2";
168     final String cf3 = "cf3";
169     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
170 
171     MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3);
172     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
173     assertEquals(3, currentHtd.getFamiliesKeys().size());
174 
175     // Test 1: Modify the table descriptor
176     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
177     htd.removeFamily(cf2.getBytes());
178 
179     long procId = ProcedureTestingUtility.submitAndWait(
180         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
181     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
182 
183     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
184     assertEquals(2, currentHtd.getFamiliesKeys().size());
185     assertFalse(currentHtd.hasFamily(cf2.getBytes()));
186 
187     // Test 2: Modify the table descriptor offline
188     UTIL.getHBaseAdmin().disableTable(tableName);
189     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
190 
191     HTableDescriptor htd2 =
192         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
193     htd2.removeFamily(cf3.getBytes());
194     // Disable Sanity check
195     htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
196 
197     long procId2 =
198         ProcedureTestingUtility.submitAndWait(procExec,
199           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
200     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
201 
202     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
203     assertEquals(1, currentHtd.getFamiliesKeys().size());
204     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
205 
206     //Removing the last family will fail
207     HTableDescriptor htd3 =
208         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
209     htd3.removeFamily(cf1.getBytes());
210     long procId3 =
211         ProcedureTestingUtility.submitAndWait(procExec,
212             new ModifyTableProcedure(procExec.getEnvironment(), htd3));
213     final ProcedureInfo result = procExec.getResult(procId3);
214     assertEquals(true, result.isFailed());
215     Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
216     assertTrue("expected DoNotRetryIOException, got " + cause,
217         cause instanceof DoNotRetryIOException);
218     assertEquals(1, currentHtd.getFamiliesKeys().size());
219     assertTrue(currentHtd.hasFamily(cf1.getBytes()));
220   }
221 
222   @Test(timeout=60000)
223   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
224     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
225     final String cf2 = "cf2";
226     final String cf3 = "cf3";
227     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
228 
229     // create the table
230     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
231       procExec, tableName, null, "cf1", cf3);
232     UTIL.getHBaseAdmin().disableTable(tableName);
233 
234     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
235     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
236 
237     // Modify multiple properties of the table.
238     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
239     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
240     htd.setCompactionEnabled(newCompactionEnableOption);
241     htd.addFamily(new HColumnDescriptor(cf2));
242     htd.removeFamily(cf3.getBytes());
243     htd.setRegionReplication(3);
244 
245     // Start the Modify procedure && kill the executor
246     long procId = procExec.submitProcedure(
247       new ModifyTableProcedure(procExec.getEnvironment(), htd));
248 
249     // Restart the executor and execute the step twice
250     int numberOfSteps = ModifyTableState.values().length;
251     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
252       procExec,
253       procId,
254       numberOfSteps,
255       ModifyTableState.values());
256 
257     // Validate descriptor
258     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
259     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
260     assertEquals(2, currentHtd.getFamiliesKeys().size());
261 
262     // cf2 should be added cf3 should be removed
263     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
264       tableName, regions, false, "cf1", cf2);
265   }
266 
267   @Test(timeout = 60000)
268   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
269     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
270     final String cf2 = "cf2";
271     final String cf3 = "cf3";
272     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
273 
274     // create the table
275     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
276       procExec, tableName, null, "cf1", cf3);
277 
278     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
279 
280     // Modify multiple properties of the table.
281     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
282     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
283     htd.setCompactionEnabled(newCompactionEnableOption);
284     htd.addFamily(new HColumnDescriptor(cf2));
285     htd.removeFamily(cf3.getBytes());
286 
287     // Start the Modify procedure && kill the executor
288     long procId = procExec.submitProcedure(
289       new ModifyTableProcedure(procExec.getEnvironment(), htd));
290 
291     // Restart the executor and execute the step twice
292     int numberOfSteps = ModifyTableState.values().length;
293     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
294       ModifyTableState.values());
295 
296     // Validate descriptor
297     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
298     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
299     assertEquals(2, currentHtd.getFamiliesKeys().size());
300     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
301     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
302 
303     // cf2 should be added cf3 should be removed
304     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
305       tableName, regions, "cf1", cf2);
306   }
307 
308   @Test(timeout = 60000)
309   public void testRollbackAndDoubleExecutionOnline() throws Exception {
310     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
311     final String familyName = "cf2";
312     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
313 
314     // create the table
315     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
316       procExec, tableName, null, "cf1");
317 
318     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
319 
320     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
321     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
322     htd.setCompactionEnabled(newCompactionEnableOption);
323     htd.addFamily(new HColumnDescriptor(familyName));
324 
325     // Start the Modify procedure && kill the executor
326     long procId = procExec.submitProcedure(
327       new ModifyTableProcedure(procExec.getEnvironment(), htd));
328 
329     // Restart the executor and rollback the step twice
330     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
331     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
332       procExec,
333       procId,
334       numberOfSteps,
335       ModifyTableState.values());
336 
337     // cf2 should not be present
338     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
339       tableName, regions, "cf1");
340   }
341 
342   @Test(timeout = 60000)
343   public void testRollbackAndDoubleExecutionOffline() throws Exception {
344     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
345     final String familyName = "cf2";
346     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
347 
348     // create the table
349     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
350       procExec, tableName, null, "cf1");
351     UTIL.getHBaseAdmin().disableTable(tableName);
352 
353     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
354     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
355 
356     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
357     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
358     htd.setCompactionEnabled(newCompactionEnableOption);
359     htd.addFamily(new HColumnDescriptor(familyName));
360     htd.setRegionReplication(3);
361 
362     // Start the Modify procedure && kill the executor
363     long procId = procExec.submitProcedure(
364       new ModifyTableProcedure(procExec.getEnvironment(), htd));
365 
366     // Restart the executor and rollback the step twice
367     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
368     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
369       procExec,
370       procId,
371       numberOfSteps,
372       ModifyTableState.values());
373 
374     // cf2 should not be present
375     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
376       tableName, regions, "cf1");
377   }
378 
379   @Test(timeout = 60000)
380   public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
381     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
382     final String familyToAddName = "cf2";
383     final String familyToRemove = "cf1";
384     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
385 
386     // create the table
387     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
388       procExec, tableName, null, familyToRemove);
389     UTIL.getHBaseAdmin().disableTable(tableName);
390 
391     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
392     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
393 
394     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
395     htd.setCompactionEnabled(!htd.isCompactionEnabled());
396     htd.addFamily(new HColumnDescriptor(familyToAddName));
397     htd.removeFamily(familyToRemove.getBytes());
398     htd.setRegionReplication(3);
399 
400     // Start the Modify procedure && kill the executor
401     long procId = procExec.submitProcedure(
402       new ModifyTableProcedure(procExec.getEnvironment(), htd));
403 
404     // Failing after MODIFY_TABLE_DELETE_FS_LAYOUT we should not trigger the rollback.
405     // NOTE: the 5 (number of MODIFY_TABLE_DELETE_FS_LAYOUT + 1 step) is hardcoded,
406     //       so you have to look at this test at least once when you add a new step.
407     int numberOfSteps = 5;
408     MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
409       procExec,
410       procId,
411       numberOfSteps,
412       ModifyTableState.values());
413 
414     // "cf2" should be added and "cf1" should be removed
415     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
416       tableName, regions, false, familyToAddName);
417   }
418 
419   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
420     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
421   }
422 }