1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23 import static org.mockito.Matchers.any;
24 import static org.mockito.Mockito.doAnswer;
25 import static org.mockito.Mockito.mock;
26 import static org.mockito.Mockito.when;
27
28 import java.io.ByteArrayOutputStream;
29 import java.io.PrintStream;
30 import java.util.ArrayList;
31
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.Cell;
36 import org.apache.hadoop.hbase.CellUtil;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.testclassification.LargeTests;
41 import org.apache.hadoop.hbase.MiniHBaseCluster;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.client.Delete;
44 import org.apache.hadoop.hbase.client.Get;
45 import org.apache.hadoop.hbase.client.Put;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.client.Table;
48 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
49 import org.apache.hadoop.hbase.mapreduce.WALPlayer.WALKeyValueMapper;
50 import org.apache.hadoop.hbase.wal.WAL;
51 import org.apache.hadoop.hbase.wal.WALKey;
52 import org.apache.hadoop.hbase.util.FSUtils;
53 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
56 import org.apache.hadoop.mapreduce.Mapper;
57 import org.apache.hadoop.mapreduce.Mapper.Context;
58 import org.junit.AfterClass;
59 import org.junit.BeforeClass;
60 import org.junit.Test;
61 import org.junit.experimental.categories.Category;
62 import org.mockito.invocation.InvocationOnMock;
63 import org.mockito.stubbing.Answer;
64
65
66
67
68 @Category(LargeTests.class)
69 public class TestWALPlayer {
70 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
71 private static MiniHBaseCluster cluster;
72 private static Path rootDir;
73 private static Path walRootDir;
74 private static FileSystem fs;
75 private static FileSystem walFs;
76 private static Configuration conf;
77
78 @BeforeClass
79 public static void beforeClass() throws Exception {
80 TEST_UTIL.setJobWithoutMRCluster();
81 conf= TEST_UTIL.getConfiguration();
82 rootDir = TEST_UTIL.createRootDir();
83 walRootDir = TEST_UTIL.createWALRootDir();
84 fs = FSUtils.getRootDirFileSystem(conf);
85 walFs = FSUtils.getWALFileSystem(conf);
86 cluster = TEST_UTIL.startMiniCluster();
87 }
88
89 @AfterClass
90 public static void afterClass() throws Exception {
91 TEST_UTIL.shutdownMiniCluster();
92 fs.delete(rootDir, true);
93 walFs.delete(walRootDir, true);
94 }
95
96
97
98
99
100 @Test
101 public void testWALPlayer() throws Exception {
102 final TableName TABLENAME1 = TableName.valueOf("testWALPlayer1");
103 final TableName TABLENAME2 = TableName.valueOf("testWALPlayer2");
104 final byte[] FAMILY = Bytes.toBytes("family");
105 final byte[] COLUMN1 = Bytes.toBytes("c1");
106 final byte[] COLUMN2 = Bytes.toBytes("c2");
107 final byte[] ROW = Bytes.toBytes("row");
108 Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
109 Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
110
111
112 Put p = new Put(ROW);
113 p.add(FAMILY, COLUMN1, COLUMN1);
114 p.add(FAMILY, COLUMN2, COLUMN2);
115 t1.put(p);
116
117 Delete d = new Delete(ROW);
118 d.deleteColumns(FAMILY, COLUMN1);
119 t1.delete(d);
120
121
122 WAL log = cluster.getRegionServer(0).getWAL(null);
123 log.rollWriter();
124 String walInputDir = new Path(cluster.getMaster().getMasterFileSystem()
125 .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
126
127 Configuration configuration= TEST_UTIL.getConfiguration();
128 WALPlayer player = new WALPlayer(configuration);
129 String optionName="_test_.name";
130 configuration.set(optionName, "1000");
131 player.setupTime(configuration, optionName);
132 assertEquals(1000,configuration.getLong(optionName,0));
133 assertEquals(0, player.run(new String[] {walInputDir, TABLENAME1.getNameAsString(),
134 TABLENAME2.getNameAsString() }));
135
136
137
138 Get g = new Get(ROW);
139 Result r = t2.get(g);
140 assertEquals(1, r.size());
141 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2));
142 }
143
144
145
146
147 @Test
148 public void testWALKeyValueMapper() throws Exception {
149 testWALKeyValueMapper(WALPlayer.TABLES_KEY);
150 }
151
152 @Test
153 public void testWALKeyValueMapperWithDeprecatedConfig() throws Exception {
154 testWALKeyValueMapper("hlog.input.tables");
155 }
156
157 private void testWALKeyValueMapper(final String tableConfigKey) throws Exception {
158 Configuration configuration = new Configuration();
159 configuration.set(tableConfigKey, "table");
160 WALKeyValueMapper mapper = new WALKeyValueMapper();
161 WALKey key = mock(WALKey.class);
162 when(key.getTablename()).thenReturn(TableName.valueOf("table"));
163 @SuppressWarnings("unchecked")
164 Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue>.Context context =
165 mock(Context.class);
166 when(context.getConfiguration()).thenReturn(configuration);
167
168 WALEdit value = mock(WALEdit.class);
169 ArrayList<Cell> values = new ArrayList<Cell>();
170 KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("q"));
171 values.add(kv1);
172 when(value.getCells()).thenReturn(values);
173 mapper.setup(context);
174
175 doAnswer(new Answer<Void>() {
176
177 @Override
178 public Void answer(InvocationOnMock invocation) throws Throwable {
179 ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArguments()[0];
180 KeyValue key = (KeyValue) invocation.getArguments()[1];
181 assertEquals("row", Bytes.toString(writer.get()));
182 assertEquals("row", Bytes.toString(key.getRow()));
183 return null;
184 }
185 }).when(context).write(any(ImmutableBytesWritable.class), any(KeyValue.class));
186
187 mapper.map(key, value, context);
188
189 }
190
191
192
193
194 @Test
195 public void testMainMethod() throws Exception {
196
197 PrintStream oldPrintStream = System.err;
198 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
199 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
200 System.setSecurityManager(newSecurityManager);
201 ByteArrayOutputStream data = new ByteArrayOutputStream();
202 String[] args = {};
203 System.setErr(new PrintStream(data));
204 try {
205 System.setErr(new PrintStream(data));
206 try {
207 WALPlayer.main(args);
208 fail("should be SecurityException");
209 } catch (SecurityException e) {
210 assertEquals(-1, newSecurityManager.getExitCode());
211 assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
212 assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
213 " <tables> [<tableMappings>]"));
214 assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
215 }
216
217 } finally {
218 System.setErr(oldPrintStream);
219 System.setSecurityManager(SECURITY_MANAGER);
220 }
221
222 }
223
224 }