1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapred;
20
21 import com.google.common.collect.Lists;
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.client.Result;
27 import org.apache.hadoop.hbase.client.Scan;
28 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
29 import org.apache.hadoop.hbase.testclassification.LargeTests;
30 import org.apache.hadoop.io.NullWritable;
31 import org.apache.hadoop.mapred.FileOutputFormat;
32 import org.apache.hadoop.mapred.JobClient;
33 import org.apache.hadoop.mapred.JobConf;
34 import org.apache.hadoop.mapred.OutputCollector;
35 import org.apache.hadoop.mapred.Reporter;
36 import org.apache.hadoop.mapred.RunningJob;
37 import org.junit.experimental.categories.Category;
38
39 import java.io.IOException;
40 import java.util.Iterator;
41 import java.util.List;
42
43 import static org.junit.Assert.assertTrue;
44
45 @Category({ LargeTests.class })
46 @edu.umd.cs.findbugs.annotations.SuppressWarnings("NM_SAME_SIMPLE_NAME_AS_SUPERCLASS")
47 public class TestMultiTableSnapshotInputFormat
48 extends org.apache.hadoop.hbase.mapreduce.TestMultiTableSnapshotInputFormat {
49
50 private static final Log LOG = LogFactory.getLog(TestMultiTableSnapshotInputFormat.class);
51
52 @Override
53 protected void runJob(String jobName, Configuration c, List<Scan> scans)
54 throws IOException, InterruptedException, ClassNotFoundException {
55 JobConf job = new JobConf(TEST_UTIL.getConfiguration());
56
57 job.setJobName(jobName);
58 job.setMapperClass(Mapper.class);
59 job.setReducerClass(Reducer.class);
60
61 TableMapReduceUtil.initMultiTableSnapshotMapperJob(getSnapshotScanMapping(scans), Mapper.class,
62 ImmutableBytesWritable.class, ImmutableBytesWritable.class, job, true, restoreDir);
63
64 TableMapReduceUtil.addDependencyJars(job);
65
66 job.setReducerClass(Reducer.class);
67 job.setNumReduceTasks(1);
68 FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
69 LOG.info("Started " + job.getJobName());
70
71 RunningJob runningJob = JobClient.runJob(job);
72 runningJob.waitForCompletion();
73 assertTrue(runningJob.isSuccessful());
74 LOG.info("After map/reduce completion - job " + jobName);
75 }
76
77 public static class Mapper extends TestMultiTableSnapshotInputFormat.ScanMapper
78 implements TableMap<ImmutableBytesWritable, ImmutableBytesWritable> {
79
80 @Override
81 public void map(ImmutableBytesWritable key, Result value,
82 OutputCollector<ImmutableBytesWritable, ImmutableBytesWritable> outputCollector,
83 Reporter reporter) throws IOException {
84 makeAssertions(key, value);
85 outputCollector.collect(key, key);
86 }
87
88
89
90
91
92
93
94
95 @Override
96 public void close() throws IOException {
97 }
98
99 @Override
100 public void configure(JobConf jobConf) {
101
102 }
103 }
104
105 public static class Reducer extends TestMultiTableSnapshotInputFormat.ScanReducer implements
106 org.apache.hadoop.mapred.Reducer<ImmutableBytesWritable, ImmutableBytesWritable,
107 NullWritable, NullWritable> {
108
109 private JobConf jobConf;
110
111 @Override
112 public void reduce(ImmutableBytesWritable key, Iterator<ImmutableBytesWritable> values,
113 OutputCollector<NullWritable, NullWritable> outputCollector, Reporter reporter)
114 throws IOException {
115 makeAssertions(key, Lists.newArrayList(values));
116 }
117
118
119
120
121
122
123
124
125 @Override
126 public void close() throws IOException {
127 super.cleanup(this.jobConf);
128 }
129
130 @Override
131 public void configure(JobConf jobConf) {
132 this.jobConf = jobConf;
133 }
134 }
135 }