View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import org.apache.commons.io.IOUtils;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.fs.FileUtil;
24  import org.apache.hadoop.fs.LocalFileSystem;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.testclassification.LargeTests;
28  import org.apache.hadoop.hbase.client.Put;
29  import org.apache.hadoop.hbase.client.Table;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
32  import org.apache.hadoop.mapreduce.Job;
33  import org.apache.hadoop.util.GenericOptionsParser;
34  import org.junit.AfterClass;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  import java.io.*;
40  
41  import static org.junit.Assert.assertTrue;
42  import static org.junit.Assert.assertEquals;
43  import static org.junit.Assert.fail;
44  
45  @Category(LargeTests.class)
46  public class TestCellCounter {
47    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48    private static final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
49    private static final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
50    private static final String FAMILY_A_STRING = "a";
51    private static final String FAMILY_B_STRING = "b";
52    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
53    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
54    private static final byte[] QUALIFIER = Bytes.toBytes("q");
55  
56    private static Path FQ_OUTPUT_DIR;
57    private static final String OUTPUT_DIR = "target" + File.separator + "test-data" + File.separator
58        + "output";
59    private static long now = System.currentTimeMillis();
60  
61    @BeforeClass
62    public static void beforeClass() throws Exception {
63      UTIL.setJobWithoutMRCluster();
64      UTIL.startMiniCluster();
65      FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
66      FileUtil.fullyDelete(new File(OUTPUT_DIR));
67    }
68  
69    @AfterClass
70    public static void afterClass() throws Exception {
71      UTIL.shutdownMiniCluster();
72    }
73  
74    /**
75     * Test CellCounter all data should print to output
76     *
77     */
78    @Test (timeout=300000)
79    public void testCellCounter() throws Exception {
80      String sourceTable = "sourceTable";
81      byte[][] families = { FAMILY_A, FAMILY_B };
82      Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
83      try{
84      Put p = new Put(ROW1);
85      p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
86      p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
87      p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
88      t.put(p);
89      p = new Put(ROW2);
90      p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
91      p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
92      p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
93      t.put(p);
94      String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "^row1" };
95      runCount(args);
96      FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
97          "part-r-00000");
98      String data = IOUtils.toString(inputStream);
99      inputStream.close();
100     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
101     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
102     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
103     assertTrue(data.contains("b;q" + "\t" + "1"));
104     assertTrue(data.contains("a;q" + "\t" + "1"));
105     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
106     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
107     }finally{
108       t.close();
109       FileUtil.fullyDelete(new File(OUTPUT_DIR));
110     }
111 
112   }
113 
114   /**
115    * Test CellCounter all data should print to output
116    */
117   @Test(timeout = 300000)
118   public void testCellCounterPrefix() throws Exception {
119     String sourceTable = "testCellCounterPrefix";
120     byte[][] families = { FAMILY_A, FAMILY_B };
121     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
122     try {
123       Put p = new Put(ROW1);
124       p.addColumn(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
125       p.addColumn(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
126       p.addColumn(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
127       t.put(p);
128       p = new Put(ROW2);
129       p.addColumn(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
130       p.addColumn(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
131       p.addColumn(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
132       t.put(p);
133       String[] args = { sourceTable, FQ_OUTPUT_DIR.toString(), ";", "\\x01row1" };
134       runCount(args);
135       FileInputStream inputStream =
136           new FileInputStream(OUTPUT_DIR + File.separator + "part-r-00000");
137       String data = IOUtils.toString(inputStream);
138       inputStream.close();
139       assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
140       assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
141       assertTrue(data.contains("Total ROWS" + "\t" + "1"));
142       assertTrue(data.contains("b;q" + "\t" + "1"));
143       assertTrue(data.contains("a;q" + "\t" + "1"));
144       assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
145       assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
146     } finally {
147       t.close();
148       FileUtil.fullyDelete(new File(OUTPUT_DIR));
149     }
150   }
151 
152   /**
153    * Test CellCounter with time range all data should print to output
154    */
155   @Test (timeout=300000)
156   public void testCellCounterStartTimeRange() throws Exception {
157     String sourceTable = "testCellCounterStartTimeRange";
158     byte[][] families = { FAMILY_A, FAMILY_B };
159     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
160     try{
161     Put p = new Put(ROW1);
162     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
163     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
164     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
165     t.put(p);
166     p = new Put(ROW2);
167     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
168     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
169     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
170     t.put(p);
171     String[] args = {
172       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--starttime=" + now,
173       "--endtime=" + now + 2 };
174     runCount(args);
175     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
176         "part-r-00000");
177     String data = IOUtils.toString(inputStream);
178     inputStream.close();
179     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
180     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
181     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
182     assertTrue(data.contains("b;q" + "\t" + "1"));
183     assertTrue(data.contains("a;q" + "\t" + "1"));
184     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
185     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
186     }finally{
187       t.close();
188       FileUtil.fullyDelete(new File(OUTPUT_DIR));
189     }
190   }
191 
192   /**
193    * Test CellCounter with time range all data should print to output
194    */
195   @Test (timeout=300000)
196   public void testCellCounteEndTimeRange() throws Exception {
197     String sourceTable = "testCellCounterEndTimeRange";
198     byte[][] families = { FAMILY_A, FAMILY_B };
199     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
200     try{
201     Put p = new Put(ROW1);
202     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
203     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
204     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
205     t.put(p);
206     p = new Put(ROW2);
207     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
208     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
209     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
210     t.put(p);
211     String[] args = {
212       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "^row1", "--endtime=" + now + 1 };
213     runCount(args);
214     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
215         "part-r-00000");
216     String data = IOUtils.toString(inputStream);
217     inputStream.close();
218     assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
219     assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "2"));
220     assertTrue(data.contains("Total ROWS" + "\t" + "1"));
221     assertTrue(data.contains("b;q" + "\t" + "1"));
222     assertTrue(data.contains("a;q" + "\t" + "1"));
223     assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
224     assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
225     }finally{
226       t.close();
227       FileUtil.fullyDelete(new File(OUTPUT_DIR));
228     }
229   }
230 
231    /**
232    * Test CellCounter with time range all data should print to output
233    */
234   @Test (timeout=300000)
235   public void testCellCounteOutOfTimeRange() throws Exception {
236     String sourceTable = "testCellCounterOutTimeRange";
237     byte[][] families = { FAMILY_A, FAMILY_B };
238     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
239     try{
240     Put p = new Put(ROW1);
241     p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
242     p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
243     p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
244     t.put(p);
245     p = new Put(ROW2);
246     p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
247     p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
248     p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
249     t.put(p);
250     String[] args = {
251       sourceTable, FQ_OUTPUT_DIR.toString(),  ";", "--starttime=" + now + 1,
252       "--endtime=" + now + 2 };
253 
254     runCount(args);
255     FileInputStream inputStream = new FileInputStream(OUTPUT_DIR + File.separator +
256         "part-r-00000");
257     String data = IOUtils.toString(inputStream);
258     inputStream.close();
259     // nothing should hace been emitted to the reducer
260     assertTrue(data.isEmpty());
261     }finally{
262       t.close();
263       FileUtil.fullyDelete(new File(OUTPUT_DIR));
264     }
265   }
266 
267 
268   private boolean runCount(String[] args) throws IOException, InterruptedException,
269       ClassNotFoundException {
270     // need to make a copy of the configuration because to make sure
271     // different temp dirs are used.
272     GenericOptionsParser opts = new GenericOptionsParser(
273         new Configuration(UTIL.getConfiguration()), args);
274     Configuration configuration = opts.getConfiguration();
275     args = opts.getRemainingArgs();
276     Job job = CellCounter.createSubmittableJob(configuration, args);
277     job.waitForCompletion(false);
278     return job.isSuccessful();
279   }
280 
281   /**
282    * Test main method of CellCounter
283    */
284   @Test (timeout=300000)
285   public void testCellCounterMain() throws Exception {
286 
287     PrintStream oldPrintStream = System.err;
288     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
289     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
290     System.setSecurityManager(newSecurityManager);
291     ByteArrayOutputStream data = new ByteArrayOutputStream();
292     String[] args = {};
293     System.setErr(new PrintStream(data));
294     try {
295       System.setErr(new PrintStream(data));
296 
297       try {
298         CellCounter.main(args);
299         fail("should be SecurityException");
300       } catch (SecurityException e) {
301         assertEquals(-1, newSecurityManager.getExitCode());
302         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
303         // should be information about usage
304         assertTrue(data.toString().contains("Usage:"));
305       }
306 
307     } finally {
308       System.setErr(oldPrintStream);
309       System.setSecurityManager(SECURITY_MANAGER);
310     }
311   }
312 
313   /**
314    * Test CellCounter for complete table all data should print to output
315    */
316   @Test(timeout = 300000)
317   public void testCellCounterForCompleteTable() throws Exception {
318     String sourceTable = "testCellCounterForCompleteTable";
319     String outputPath = OUTPUT_DIR + sourceTable;
320     LocalFileSystem localFileSystem = new LocalFileSystem();
321     Path outputDir =
322         new Path(outputPath).makeQualified(localFileSystem.getUri(),
323           localFileSystem.getWorkingDirectory());
324     byte[][] families = { FAMILY_A, FAMILY_B };
325     Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families);
326     try {
327       Put p = new Put(ROW1);
328       p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11"));
329       p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12"));
330       p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13"));
331       t.put(p);
332       p = new Put(ROW2);
333       p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21"));
334       p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22"));
335       p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23"));
336       t.put(p);
337       String[] args = { sourceTable, outputDir.toString(), ";" };
338       runCount(args);
339       FileInputStream inputStream =
340           new FileInputStream(outputPath + File.separator + "part-r-00000");
341       String data = IOUtils.toString(inputStream);
342       inputStream.close();
343       assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2"));
344       assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4"));
345       assertTrue(data.contains("Total ROWS" + "\t" + "2"));
346       assertTrue(data.contains("b;q" + "\t" + "2"));
347       assertTrue(data.contains("a;q" + "\t" + "2"));
348       assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1"));
349       assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1"));
350       assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1"));
351       assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1"));
352     } finally {
353       t.close();
354       FileUtil.fullyDelete(new File(outputPath));
355     }
356   }
357 
358   @Test (timeout=300000)
359   public void TestCellCounterWithoutOutputDir() throws Exception {
360     PrintStream oldPrintStream = System.err;
361     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
362     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
363     System.setSecurityManager(newSecurityManager);
364     ByteArrayOutputStream data = new ByteArrayOutputStream();
365     String[] args = {"tableName"};
366     System.setErr(new PrintStream(data));
367     try {
368       System.setErr(new PrintStream(data));
369       try {
370         CellCounter.main(args);
371         fail("should be SecurityException");
372       } catch (SecurityException e) {
373         assertEquals(-1, newSecurityManager.getExitCode());
374         assertTrue(data.toString().contains("ERROR: Wrong number of parameters:"));
375         // should be information about usage
376         assertTrue(data.toString().contains("Usage:"));
377       }
378 
379     } finally {
380       System.setErr(oldPrintStream);
381       System.setSecurityManager(SECURITY_MANAGER);
382     }
383   }
384 }