View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.Arrays;
30  import java.util.List;
31  import java.util.Map;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.Cell;
39  import org.apache.hadoop.hbase.Coprocessor;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HBaseTestingUtility;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HRegionInfo;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.KeyValue;
47  import org.apache.hadoop.hbase.testclassification.MediumTests;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.regionserver.HRegion;
51  import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
52  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
53  import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
54  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
55  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
56  import org.apache.hadoop.hbase.wal.WAL;
57  import org.apache.hadoop.hbase.wal.WALFactory;
58  import org.apache.hadoop.hbase.wal.WALKey;
59  import org.apache.hadoop.hbase.wal.WALSplitter;
60  import org.apache.hadoop.hbase.security.User;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.EnvironmentEdge;
63  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
64  import org.apache.hadoop.hbase.util.FSUtils;
65  import org.junit.After;
66  import org.junit.AfterClass;
67  import org.junit.Before;
68  import org.junit.BeforeClass;
69  import org.junit.Rule;
70  import org.junit.Test;
71  import org.junit.rules.TestName;
72  import org.junit.experimental.categories.Category;
73  
74  /**
75   * Tests invocation of the
76   * {@link org.apache.hadoop.hbase.coprocessor.MasterObserver} interface hooks at
77   * all appropriate times during normal HMaster operations.
78   */
79  @Category(MediumTests.class)
80  public class TestWALObserver {
81    private static final Log LOG = LogFactory.getLog(TestWALObserver.class);
82    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
83  
84    private static byte[] TEST_TABLE = Bytes.toBytes("observedTable");
85    private static byte[][] TEST_FAMILY = { Bytes.toBytes("fam1"),
86        Bytes.toBytes("fam2"), Bytes.toBytes("fam3"), };
87    private static byte[][] TEST_QUALIFIER = { Bytes.toBytes("q1"),
88        Bytes.toBytes("q2"), Bytes.toBytes("q3"), };
89    private static byte[][] TEST_VALUE = { Bytes.toBytes("v1"),
90        Bytes.toBytes("v2"), Bytes.toBytes("v3"), };
91    private static byte[] TEST_ROW = Bytes.toBytes("testRow");
92  
93    @Rule
94    public TestName currentTest = new TestName();
95  
96    private Configuration conf;
97    private FileSystem fs;
98    private Path dir;
99    private Path hbaseRootDir;
100   private Path hbaseWALRootDir;
101   private String logName;
102   private Path oldLogDir;
103   private Path logDir;
104   private WALFactory wals;
105 
106   @BeforeClass
107   public static void setupBeforeClass() throws Exception {
108     Configuration conf = TEST_UTIL.getConfiguration();
109     conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
110         SampleRegionWALObserver.class.getName(), SampleRegionWALObserver.Legacy.class.getName());
111     conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
112         SampleRegionWALObserver.class.getName());
113     conf.setBoolean("dfs.support.append", true);
114     conf.setInt("dfs.client.block.recovery.retries", 2);
115 
116     TEST_UTIL.startMiniCluster(1);
117     Path hbaseRootDir = TEST_UTIL.getDFSCluster().getFileSystem()
118         .makeQualified(new Path("/hbase"));
119     Path hbaseWALRootDir = TEST_UTIL.getDFSCluster().getFileSystem()
120             .makeQualified(new Path("/hbaseLogRoot"));
121     LOG.info("hbase.rootdir=" + hbaseRootDir);
122     FSUtils.setRootDir(conf, hbaseRootDir);
123     FSUtils.setWALRootDir(conf, hbaseWALRootDir);
124   }
125 
126   @AfterClass
127   public static void teardownAfterClass() throws Exception {
128     TEST_UTIL.shutdownMiniCluster();
129   }
130 
131   @Before
132   public void setUp() throws Exception {
133     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
134     // this.cluster = TEST_UTIL.getDFSCluster();
135     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
136     this.hbaseRootDir = FSUtils.getRootDir(conf);
137     this.hbaseWALRootDir = FSUtils.getWALRootDir(conf);
138     this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName());
139     this.oldLogDir = new Path(this.hbaseWALRootDir,
140         HConstants.HREGION_OLDLOGDIR_NAME);
141     this.logDir = new Path(this.hbaseWALRootDir,
142         DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
143     this.logName = HConstants.HREGION_LOGDIR_NAME;
144 
145     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
146       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
147     }
148     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseWALRootDir)) {
149       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true);
150     }
151     this.wals = new WALFactory(conf, null, currentTest.getMethodName());
152   }
153 
154   @After
155   public void tearDown() throws Exception {
156     try {
157       wals.shutdown();
158     } catch (IOException exception) {
159       // one of our tests splits out from under our wals.
160       LOG.warn("Ignoring failure to close wal factory. " + exception.getMessage());
161       LOG.debug("details of failure to close wal factory.", exception);
162     }
163     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
164     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true);
165   }
166 
167   /**
168    * Test WAL write behavior with WALObserver. The coprocessor monitors a
169    * WALEdit written to WAL, and ignore, modify, and add KeyValue's for the
170    * WALEdit.
171    */
172   @Test
173   public void testWALObserverWriteToWAL() throws Exception {
174     final WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
175     verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.class), false);
176   }
177 
178   /**
179    * Test WAL write behavior with WALObserver. The coprocessor monitors a
180    * WALEdit written to WAL, and ignore, modify, and add KeyValue's for the
181    * WALEdit.
182    */
183   @Test
184   public void testLegacyWALObserverWriteToWAL() throws Exception {
185     final WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
186     verifyWritesSeen(log, getCoprocessor(log, SampleRegionWALObserver.Legacy.class), true);
187   }
188 
189   private void verifyWritesSeen(final WAL log, final SampleRegionWALObserver cp,
190       final boolean seesLegacy) throws Exception {
191     HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
192     final HTableDescriptor htd = createBasic3FamilyHTD(Bytes
193         .toString(TEST_TABLE));
194 
195     Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
196     deleteDir(basedir);
197     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
198 
199     // TEST_FAMILY[0] shall be removed from WALEdit.
200     // TEST_FAMILY[1] value shall be changed.
201     // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put.
202     cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0],
203         TEST_FAMILY[1], TEST_QUALIFIER[1], TEST_FAMILY[2], TEST_QUALIFIER[2]);
204 
205     assertFalse(cp.isPreWALWriteCalled());
206     assertFalse(cp.isPostWALWriteCalled());
207     assertFalse(cp.isPreWALWriteDeprecatedCalled());
208     assertFalse(cp.isPostWALWriteDeprecatedCalled());
209 
210     // TEST_FAMILY[2] is not in the put, however it shall be added by the tested
211     // coprocessor.
212     // Use a Put to create familyMap.
213     Put p = creatPutWith2Families(TEST_ROW);
214 
215     Map<byte[], List<Cell>> familyMap = p.getFamilyCellMap();
216     WALEdit edit = new WALEdit();
217     addFamilyMapToWALEdit(familyMap, edit);
218 
219     boolean foundFamily0 = false;
220     boolean foundFamily2 = false;
221     boolean modifiedFamily1 = false;
222 
223     List<Cell> cells = edit.getCells();
224 
225     for (Cell cell : cells) {
226       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[0])) {
227         foundFamily0 = true;
228       }
229       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[2])) {
230         foundFamily2 = true;
231       }
232       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[1])) {
233         if (!Arrays.equals(cell.getValue(), TEST_VALUE[1])) {
234           modifiedFamily1 = true;
235         }
236       }
237     }
238     assertTrue(foundFamily0);
239     assertFalse(foundFamily2);
240     assertFalse(modifiedFamily1);
241 
242     // it's where WAL write cp should occur.
243     long now = EnvironmentEdgeManager.currentTime();
244     // we use HLogKey here instead of WALKey directly to support legacy coprocessors.
245     long txid = log.append(htd, hri,
246         new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now,
247             new MultiVersionConcurrencyControl()),
248         edit, true);
249     log.sync(txid);
250 
251     // the edit shall have been change now by the coprocessor.
252     foundFamily0 = false;
253     foundFamily2 = false;
254     modifiedFamily1 = false;
255     for (Cell cell : cells) {
256       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[0])) {
257         foundFamily0 = true;
258       }
259       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[2])) {
260         foundFamily2 = true;
261       }
262       if (Arrays.equals(cell.getFamily(), TEST_FAMILY[1])) {
263         if (!Arrays.equals(cell.getValue(), TEST_VALUE[1])) {
264           modifiedFamily1 = true;
265         }
266       }
267     }
268     assertFalse(foundFamily0);
269     assertTrue(foundFamily2);
270     assertTrue(modifiedFamily1);
271 
272     assertTrue(cp.isPreWALWriteCalled());
273     assertTrue(cp.isPostWALWriteCalled());
274     assertEquals(seesLegacy, cp.isPreWALWriteDeprecatedCalled());
275     assertEquals(seesLegacy, cp.isPostWALWriteDeprecatedCalled());
276   }
277 
278   @Test
279   public void testNonLegacyWALKeysDoNotExplode() throws Exception {
280     TableName tableName = TableName.valueOf(TEST_TABLE);
281     final HTableDescriptor htd = createBasic3FamilyHTD(Bytes
282         .toString(TEST_TABLE));
283     final HRegionInfo hri = new HRegionInfo(tableName, null, null);
284     MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
285 
286     fs.mkdirs(new Path(FSUtils.getTableDir(hbaseRootDir, tableName), hri.getEncodedName()));
287 
288     final Configuration newConf = HBaseConfiguration.create(this.conf);
289 
290     final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
291     final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class);
292     newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
293     final SampleRegionWALObserver oldApi = getCoprocessor(wal,
294         SampleRegionWALObserver.Legacy.class);
295     oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
296 
297     LOG.debug("ensuring wal entries haven't happened before we start");
298     assertFalse(newApi.isPreWALWriteCalled());
299     assertFalse(newApi.isPostWALWriteCalled());
300     assertFalse(newApi.isPreWALWriteDeprecatedCalled());
301     assertFalse(newApi.isPostWALWriteDeprecatedCalled());
302     assertFalse(oldApi.isPreWALWriteCalled());
303     assertFalse(oldApi.isPostWALWriteCalled());
304     assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
305     assertFalse(oldApi.isPostWALWriteDeprecatedCalled());
306 
307     LOG.debug("writing to WAL with non-legacy keys.");
308     final int countPerFamily = 5;
309     for (HColumnDescriptor hcd : htd.getFamilies()) {
310       addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
311           EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
312     }
313 
314     LOG.debug("Verify that only the non-legacy CP saw edits.");
315     assertTrue(newApi.isPreWALWriteCalled());
316     assertTrue(newApi.isPostWALWriteCalled());
317     assertFalse(newApi.isPreWALWriteDeprecatedCalled());
318     assertFalse(newApi.isPostWALWriteDeprecatedCalled());
319     // wish we could test that the log message happened :/
320     assertFalse(oldApi.isPreWALWriteCalled());
321     assertFalse(oldApi.isPostWALWriteCalled());
322     assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
323     assertFalse(oldApi.isPostWALWriteDeprecatedCalled());
324 
325     LOG.debug("reseting cp state.");
326     newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
327     oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
328 
329     LOG.debug("write a log edit that supports legacy cps.");
330     final long now = EnvironmentEdgeManager.currentTime();
331     final WALKey legacyKey = new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc);
332     final WALEdit edit = new WALEdit();
333     final byte[] nonce = Bytes.toBytes("1772");
334     edit.add(new KeyValue(TEST_ROW, TEST_FAMILY[0], nonce, now, nonce));
335     final long txid = wal.append(htd, hri, legacyKey, edit, true);
336     wal.sync(txid);
337 
338     LOG.debug("Make sure legacy cps can see supported edits after having been skipped.");
339     assertTrue("non-legacy WALObserver didn't see pre-write.", newApi.isPreWALWriteCalled());
340     assertTrue("non-legacy WALObserver didn't see post-write.", newApi.isPostWALWriteCalled());
341     assertFalse("non-legacy WALObserver shouldn't have seen legacy pre-write.",
342         newApi.isPreWALWriteDeprecatedCalled());
343     assertFalse("non-legacy WALObserver shouldn't have seen legacy post-write.",
344         newApi.isPostWALWriteDeprecatedCalled());
345     assertTrue("legacy WALObserver didn't see pre-write.", oldApi.isPreWALWriteCalled());
346     assertTrue("legacy WALObserver didn't see post-write.", oldApi.isPostWALWriteCalled());
347     assertTrue("legacy WALObserver didn't see legacy pre-write.",
348         oldApi.isPreWALWriteDeprecatedCalled());
349     assertTrue("legacy WALObserver didn't see legacy post-write.",
350         oldApi.isPostWALWriteDeprecatedCalled());
351   }
352 
353   /**
354    * Coprocessors shouldn't get notice of empty waledits.
355    */
356   @Test
357   public void testEmptyWALEditAreNotSeen() throws Exception {
358     final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
359     final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
360     final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
361 
362     WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
363     try {
364       SampleRegionWALObserver cp = getCoprocessor(log, SampleRegionWALObserver.class);
365 
366       cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);
367 
368       assertFalse(cp.isPreWALWriteCalled());
369       assertFalse(cp.isPostWALWriteCalled());
370 
371       final long now = EnvironmentEdgeManager.currentTime();
372       long txid = log.append(htd, hri,
373           new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc),
374           new WALEdit(), true);
375       log.sync(txid);
376 
377       assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
378       assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
379     } finally {
380       log.close();
381     }
382   }
383 
384   /**
385    * Test WAL replay behavior with WALObserver.
386    */
387   @Test
388   public void testWALCoprocessorReplay() throws Exception {
389     // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
390     // ultimately called by HRegion::initialize()
391     TableName tableName = TableName.valueOf("testWALCoprocessorReplay");
392     final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
393     MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
394     // final HRegionInfo hri =
395     // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
396     // final HRegionInfo hri1 =
397     // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
398     final HRegionInfo hri = new HRegionInfo(tableName, null, null);
399 
400     final Path basedir =
401         FSUtils.getTableDir(this.hbaseRootDir, tableName);
402     deleteDir(basedir);
403     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
404 
405     final Configuration newConf = HBaseConfiguration.create(this.conf);
406 
407     // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
408     WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
409     // Put p = creatPutWith2Families(TEST_ROW);
410     WALEdit edit = new WALEdit();
411     long now = EnvironmentEdgeManager.currentTime();
412     // addFamilyMapToWALEdit(p.getFamilyMap(), edit);
413     final int countPerFamily = 1000;
414     // for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
415     for (HColumnDescriptor hcd : htd.getFamilies()) {
416       addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
417           EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
418     }
419     wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc), edit, true);
420     // sync to fs.
421     wal.sync();
422 
423     User user = HBaseTestingUtility.getDifferentUser(newConf,
424         ".replay.wal.secondtime");
425     user.runAs(new PrivilegedExceptionAction() {
426       public Object run() throws Exception {
427         Path p = runWALSplit(newConf);
428         LOG.info("WALSplit path == " + p);
429         FileSystem newFS = FileSystem.get(newConf);
430         // Make a new wal for new region open.
431         final WALFactory wals2 = new WALFactory(conf, null, currentTest.getMethodName()+"2");
432         WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION, null);;
433         HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir,
434             hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
435         long seqid2 = region.getOpenSeqNum();
436 
437         SampleRegionWALObserver cp2 =
438           (SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor(
439               SampleRegionWALObserver.class.getName());
440         // TODO: asserting here is problematic.
441         assertNotNull(cp2);
442         assertTrue(cp2.isPreWALRestoreCalled());
443         assertTrue(cp2.isPostWALRestoreCalled());
444         assertFalse(cp2.isPreWALRestoreDeprecatedCalled());
445         assertFalse(cp2.isPostWALRestoreDeprecatedCalled());
446         region.close();
447         wals2.close();
448         return null;
449       }
450     });
451   }
452 
453   /**
454    * Test to see CP loaded successfully or not. There is a duplication at
455    * TestHLog, but the purpose of that one is to see whether the loaded CP will
456    * impact existing WAL tests or not.
457    */
458   @Test
459   public void testWALObserverLoaded() throws Exception {
460     WAL log = wals.getWAL(UNSPECIFIED_REGION, null);
461     assertNotNull(getCoprocessor(log, SampleRegionWALObserver.class));
462   }
463 
464   @Test
465   public void testWALObserverRoll() throws Exception {
466     final WAL wal = wals.getWAL(UNSPECIFIED_REGION, null);
467     final SampleRegionWALObserver cp = getCoprocessor(wal, SampleRegionWALObserver.class);
468     cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);
469 
470     assertFalse(cp.isPreWALRollCalled());
471     assertFalse(cp.isPostWALRollCalled());
472 
473     wal.rollWriter(true);
474     assertTrue(cp.isPreWALRollCalled());
475     assertTrue(cp.isPostWALRollCalled());
476   }
477 
478   private SampleRegionWALObserver getCoprocessor(WAL wal,
479       Class<? extends SampleRegionWALObserver> clazz) throws Exception {
480     WALCoprocessorHost host = wal.getCoprocessorHost();
481     Coprocessor c = host.findCoprocessor(clazz.getName());
482     return (SampleRegionWALObserver) c;
483   }
484 
485   /*
486    * Creates an HRI around an HTD that has <code>tableName</code> and three
487    * column families named.
488    * 
489    * @param tableName Name of table to use when we create HTableDescriptor.
490    */
491   private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
492     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
493 
494     for (int i = 0; i < TEST_FAMILY.length; i++) {
495       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
496       htd.addFamily(a);
497     }
498     return new HRegionInfo(htd.getTableName(), null, null, false);
499   }
500 
501   /*
502    * @param p Directory to cleanup
503    */
504   private void deleteDir(final Path p) throws IOException {
505     if (this.fs.exists(p)) {
506       if (!this.fs.delete(p, true)) {
507         throw new IOException("Failed remove of " + p);
508       }
509     }
510   }
511 
512   private Put creatPutWith2Families(byte[] row) throws IOException {
513     Put p = new Put(row);
514     for (int i = 0; i < TEST_FAMILY.length - 1; i++) {
515       p.add(TEST_FAMILY[i], TEST_QUALIFIER[i], TEST_VALUE[i]);
516     }
517     return p;
518   }
519 
520   /**
521    * Copied from HRegion.
522    * 
523    * @param familyMap
524    *          map of family->edits
525    * @param walEdit
526    *          the destination entry to append into
527    */
528   private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
529       WALEdit walEdit) {
530     for (List<Cell> edits : familyMap.values()) {
531       for (Cell cell : edits) {
532         // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO.
533         walEdit.add(cell);
534       }
535     }
536   }
537 
538   private Path runWALSplit(final Configuration c) throws IOException {
539     List<Path> splits = WALSplitter.split(
540       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c, wals);
541     // Split should generate only 1 file since there's only 1 region
542     assertEquals(1, splits.size());
543     // Make sure the file exists
544     assertTrue(fs.exists(splits.get(0)));
545     LOG.info("Split file=" + splits.get(0));
546     return splits.get(0);
547   }
548 
549   private static final byte[] UNSPECIFIED_REGION = new byte[]{};
550 
551   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
552       final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
553       final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
554     String familyStr = Bytes.toString(family);
555     long txid = -1;
556     for (int j = 0; j < count; j++) {
557       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
558       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
559       WALEdit edit = new WALEdit();
560       edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
561       // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
562       // about legacy coprocessors
563       txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
564           ee.currentTime(), mvcc), edit, true);
565     }
566     if (-1 != txid) {
567       wal.sync(txid);
568     }
569   }
570 
571   private HTableDescriptor getBasic3FamilyHTableDescriptor(
572       final TableName tableName) {
573     HTableDescriptor htd = new HTableDescriptor(tableName);
574 
575     for (int i = 0; i < TEST_FAMILY.length; i++) {
576       HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]);
577       htd.addFamily(a);
578     }
579     return htd;
580   }
581 
582   private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
583     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
584     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
585     htd.addFamily(a);
586     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
587     htd.addFamily(b);
588     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
589     htd.addFamily(c);
590     return htd;
591   }
592 }