1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.ByteArrayOutputStream;
27 import java.io.IOException;
28 import java.io.PrintStream;
29
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellUtil;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.client.Table;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
42 import org.apache.hadoop.util.ToolRunner;
43 import org.junit.AfterClass;
44 import org.junit.Assert;
45 import org.junit.BeforeClass;
46 import org.junit.Test;
47 import org.junit.experimental.categories.Category;
48
49
50
51
52 @Category(LargeTests.class)
53 public class TestCopyTable {
54 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55 private static final byte[] ROW1 = Bytes.toBytes("row1");
56 private static final byte[] ROW2 = Bytes.toBytes("row2");
57 private static final String FAMILY_A_STRING = "a";
58 private static final String FAMILY_B_STRING = "b";
59 private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
60 private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
61 private static final byte[] QUALIFIER = Bytes.toBytes("q");
62
63
64 @BeforeClass
65 public static void beforeClass() throws Exception {
66 TEST_UTIL.setJobWithoutMRCluster();
67 TEST_UTIL.startMiniCluster(3);
68 }
69
70 @AfterClass
71 public static void afterClass() throws Exception {
72 TEST_UTIL.shutdownMiniCluster();
73 }
74
75 private void doCopyTableTest(boolean bulkload) throws Exception {
76 final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
77 final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
78 final byte[] FAMILY = Bytes.toBytes("family");
79 final byte[] COLUMN1 = Bytes.toBytes("c1");
80
81 try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
82 Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);) {
83
84 loadData(t1, FAMILY, COLUMN1);
85
86 CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
87 int code;
88 if (bulkload) {
89 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
90 copy, new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
91 "--bulkload", TABLENAME1.getNameAsString() });
92 } else {
93 code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
94 copy, new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
95 TABLENAME1.getNameAsString() });
96 }
97 assertEquals("copy job failed", 0, code);
98
99
100 verifyRows(t2, FAMILY, COLUMN1);
101 } finally {
102 TEST_UTIL.deleteTable(TABLENAME1);
103 TEST_UTIL.deleteTable(TABLENAME2);
104 }
105 }
106
107
108
109
110
111 @Test
112 public void testCopyTable() throws Exception {
113 doCopyTableTest(false);
114 }
115
116
117
118
119 @Test
120 public void testCopyTableWithBulkload() throws Exception {
121 doCopyTableTest(true);
122 }
123
124 @Test
125 public void testStartStopRow() throws Exception {
126 final TableName TABLENAME1 = TableName.valueOf("testStartStopRow1");
127 final TableName TABLENAME2 = TableName.valueOf("testStartStopRow2");
128 final byte[] FAMILY = Bytes.toBytes("family");
129 final byte[] COLUMN1 = Bytes.toBytes("c1");
130 final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
131 final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
132 final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
133
134 Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
135 Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
136
137
138 Put p = new Put(ROW0);
139 p.add(FAMILY, COLUMN1, COLUMN1);
140 t1.put(p);
141 p = new Put(ROW1);
142 p.add(FAMILY, COLUMN1, COLUMN1);
143 t1.put(p);
144 p = new Put(ROW2);
145 p.add(FAMILY, COLUMN1, COLUMN1);
146 t1.put(p);
147
148 CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
149 assertEquals(
150 0,
151 copy.run(new String[] { "--new.name=" + TABLENAME2, "--startrow=\\x01row1",
152 "--stoprow=\\x01row2", TABLENAME1.getNameAsString() }));
153
154
155
156 Get g = new Get(ROW1);
157 Result r = t2.get(g);
158 assertEquals(1, r.size());
159 assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
160
161 g = new Get(ROW0);
162 r = t2.get(g);
163 assertEquals(0, r.size());
164
165 g = new Get(ROW2);
166 r = t2.get(g);
167 assertEquals(0, r.size());
168
169 t1.close();
170 t2.close();
171 TEST_UTIL.deleteTable(TABLENAME1);
172 TEST_UTIL.deleteTable(TABLENAME2);
173 }
174
175
176
177
178 @Test
179 public void testRenameFamily() throws Exception {
180 String sourceTable = "sourceTable";
181 String targetTable = "targetTable";
182
183 byte[][] families = { FAMILY_A, FAMILY_B };
184
185 Table t = TEST_UTIL.createTable(Bytes.toBytes(sourceTable), families);
186 Table t2 = TEST_UTIL.createTable(Bytes.toBytes(targetTable), families);
187 Put p = new Put(ROW1);
188 p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data11"));
189 p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Data12"));
190 p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data13"));
191 t.put(p);
192 p = new Put(ROW2);
193 p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
194 p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
195 p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
196 t.put(p);
197
198 long currentTime = System.currentTimeMillis();
199 String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
200 "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
201 "--versions=1", sourceTable };
202 assertNull(t2.get(new Get(ROW1)).getRow());
203
204 assertTrue(runCopy(args));
205
206 assertNotNull(t2.get(new Get(ROW1)).getRow());
207 Result res = t2.get(new Get(ROW1));
208 byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
209 assertEquals("Data13", new String(b1));
210 assertNotNull(t2.get(new Get(ROW2)).getRow());
211 res = t2.get(new Get(ROW2));
212 b1 = res.getValue(FAMILY_A, QUALIFIER);
213
214 assertNull(b1);
215
216 }
217
218
219
220
221 @Test
222 public void testMainMethod() throws Exception {
223 String[] emptyArgs = { "-h" };
224 PrintStream oldWriter = System.err;
225 ByteArrayOutputStream data = new ByteArrayOutputStream();
226 PrintStream writer = new PrintStream(data);
227 System.setErr(writer);
228 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
229 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
230 System.setSecurityManager(newSecurityManager);
231 try {
232 CopyTable.main(emptyArgs);
233 fail("should be exit");
234 } catch (SecurityException e) {
235 assertEquals(1, newSecurityManager.getExitCode());
236 } finally {
237 System.setErr(oldWriter);
238 System.setSecurityManager(SECURITY_MANAGER);
239 }
240 assertTrue(data.toString().contains("rs.class"));
241
242 assertTrue(data.toString().contains("Usage:"));
243 }
244
245 private boolean runCopy(String[] args) throws Exception {
246 CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
247 int code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, args);
248 return code == 0;
249 }
250
251 private void loadData(Table t, byte[] family, byte[] column) throws IOException {
252 for (int i = 0; i < 10; i++) {
253 byte[] row = Bytes.toBytes("row" + i);
254 Put p = new Put(row);
255 p.addColumn(family, column, row);
256 t.put(p);
257 }
258 }
259
260 private void verifyRows(Table t, byte[] family, byte[] column) throws IOException {
261 for (int i = 0; i < 10; i++) {
262 byte[] row = Bytes.toBytes("row" + i);
263 Get g = new Get(row).addFamily(family);
264 Result r = t.get(g);
265 Assert.assertNotNull(r);
266 Assert.assertEquals(1, r.size());
267 Cell cell = r.rawCells()[0];
268 Assert.assertTrue(CellUtil.matchingQualifier(cell, column));
269 Assert.assertEquals(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(),
270 cell.getValueLength(), row, 0, row.length), 0);
271 }
272 }
273
274 private void testCopyTableBySnapshot(String tablePrefix, boolean bulkLoad)
275 throws Exception {
276 TableName table1 = TableName.valueOf(tablePrefix + 1);
277 TableName table2 = TableName.valueOf(tablePrefix + 2);
278 Table t1 = TEST_UTIL.createTable(table1, FAMILY_A);
279 Table t2 = TEST_UTIL.createTable(table2, FAMILY_A);
280 loadData(t1, FAMILY_A, Bytes.toBytes("qualifier"));
281 String snapshot = tablePrefix + "_snapshot";
282 TEST_UTIL.getHBaseAdmin().snapshot(snapshot, table1);
283 boolean success;
284 if (bulkLoad) {
285 success =
286 runCopy(new String[] { "--snapshot", "--new.name=" + table2, "--bulkload", snapshot });
287 } else {
288 success = runCopy(new String[] { "--snapshot", "--new.name=" + table2, snapshot });
289 }
290 Assert.assertTrue(success);
291 verifyRows(t2, FAMILY_A, Bytes.toBytes("qualifier"));
292 }
293
294 @Test
295 public void testLoadingSnapshotToTable() throws Exception {
296 testCopyTableBySnapshot("testLoadingSnapshotToTable", false);
297 }
298
299 @Test
300 public void testLoadingSnapshotAndBulkLoadToTable() throws Exception {
301 testCopyTableBySnapshot("testLoadingSnapshotAndBulkLoadToTable", true);
302 }
303
304 @Test
305 public void testLoadingSnapshotToRemoteCluster() throws Exception {
306 Assert.assertFalse(runCopy(
307 new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase", "sourceSnapshotName" }));
308 }
309
310 @Test
311 public void testLoadingSnapshotWithoutSnapshotName() throws Exception {
312 Assert.assertFalse(runCopy(new String[] { "--snapshot", "--peerAdr=hbase://remoteHBase" }));
313 }
314
315 @Test
316 public void testLoadingSnapshotWithoutDestTable() throws Exception {
317 Assert.assertFalse(runCopy(new String[] { "--snapshot", "sourceSnapshotName" }));
318 }
319 }