View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.io.PrintStream;
23  import java.lang.reflect.Constructor;
24  import java.math.BigDecimal;
25  import java.math.MathContext;
26  import java.text.DecimalFormat;
27  import java.text.SimpleDateFormat;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Date;
31  import java.util.LinkedList;
32  import java.util.Locale;
33  import java.util.Map;
34  import java.util.Queue;
35  import java.util.Random;
36  import java.util.TreeMap;
37  import java.util.NoSuchElementException;
38  import java.util.concurrent.Callable;
39  import java.util.concurrent.ExecutionException;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.Executors;
42  import java.util.concurrent.Future;
43  
44  import com.google.common.base.Objects;
45  import com.google.common.util.concurrent.ThreadFactoryBuilder;
46  
47  import org.apache.commons.lang.StringUtils;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.conf.Configuration;
51  import org.apache.hadoop.conf.Configured;
52  import org.apache.hadoop.fs.FileSystem;
53  import org.apache.hadoop.fs.Path;
54  import org.apache.hadoop.hbase.classification.InterfaceAudience;
55  import org.apache.hadoop.hbase.client.Admin;
56  import org.apache.hadoop.hbase.client.Append;
57  import org.apache.hadoop.hbase.client.BufferedMutator;
58  import org.apache.hadoop.hbase.client.BufferedMutatorParams;
59  import org.apache.hadoop.hbase.client.Connection;
60  import org.apache.hadoop.hbase.client.ConnectionFactory;
61  import org.apache.hadoop.hbase.client.Consistency;
62  import org.apache.hadoop.hbase.client.Delete;
63  import org.apache.hadoop.hbase.client.Durability;
64  import org.apache.hadoop.hbase.client.Get;
65  import org.apache.hadoop.hbase.client.Increment;
66  import org.apache.hadoop.hbase.client.Put;
67  import org.apache.hadoop.hbase.client.Result;
68  import org.apache.hadoop.hbase.client.ResultScanner;
69  import org.apache.hadoop.hbase.client.RowMutations;
70  import org.apache.hadoop.hbase.client.Scan;
71  import org.apache.hadoop.hbase.client.Table;
72  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
73  import org.apache.hadoop.hbase.filter.BinaryComparator;
74  import org.apache.hadoop.hbase.filter.CompareFilter;
75  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
76  import org.apache.hadoop.hbase.filter.Filter;
77  import org.apache.hadoop.hbase.filter.FilterAllFilter;
78  import org.apache.hadoop.hbase.filter.FilterList;
79  import org.apache.hadoop.hbase.filter.PageFilter;
80  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
81  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
82  import org.apache.hadoop.hbase.io.compress.Compression;
83  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
84  import org.apache.hadoop.hbase.io.hfile.RandomDistribution;
85  import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
86  import org.apache.hadoop.hbase.regionserver.BloomType;
87  import org.apache.hadoop.hbase.trace.HBaseHTraceConfiguration;
88  import org.apache.hadoop.hbase.trace.SpanReceiverHost;
89  import org.apache.hadoop.hbase.util.*;
90  import org.apache.hadoop.io.LongWritable;
91  import org.apache.hadoop.io.Text;
92  import org.apache.hadoop.mapreduce.Job;
93  import org.apache.hadoop.mapreduce.Mapper;
94  import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
95  import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
96  import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
97  import org.apache.hadoop.util.Tool;
98  import org.apache.hadoop.util.ToolRunner;
99  
100 import com.yammer.metrics.core.Histogram;
101 import com.yammer.metrics.stats.UniformSample;
102 
103 import org.apache.hbase.thirdparty.com.google.gson.Gson;
104 import org.apache.htrace.Sampler;
105 import org.apache.htrace.Trace;
106 import org.apache.htrace.TraceScope;
107 import org.apache.htrace.impl.ProbabilitySampler;
108 
109 /**
110  * Script used evaluating HBase performance and scalability.  Runs a HBase
111  * client that steps through one of a set of hardcoded tests or 'experiments'
112  * (e.g. a random reads test, a random writes test, etc.). Pass on the
113  * command-line which test to run and how many clients are participating in
114  * this experiment. Run {@code PerformanceEvaluation --help} to obtain usage.
115  *
116  * <p>This class sets up and runs the evaluation programs described in
117  * Section 7, <i>Performance Evaluation</i>, of the <a
118  * href="http://labs.google.com/papers/bigtable.html">Bigtable</a>
119  * paper, pages 8-10.
120  *
121  * <p>By default, runs as a mapreduce job where each mapper runs a single test
122  * client. Can also run as a non-mapreduce, multithreaded application by
123  * specifying {@code --nomapred}. Each client does about 1GB of data, unless
124  * specified otherwise.
125  */
126 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
127 public class PerformanceEvaluation extends Configured implements Tool {
128   private static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
129   private static final Gson GSON = GsonUtil.createGson().create();
130 
131   public static final String TABLE_NAME = "TestTable";
132   public static final String FAMILY_NAME_BASE = "info";
133   public static final byte[] FAMILY_ZERO = Bytes.toBytes("info0");
134   public static final byte[] COLUMN_ZERO = Bytes.toBytes("" + 0);
135   public static final int DEFAULT_VALUE_LENGTH = 1000;
136   public static final int ROW_LENGTH = 26;
137 
138   private static final int ONE_GB = 1024 * 1024 * 1000;
139   private static final int DEFAULT_ROWS_PER_GB = ONE_GB / DEFAULT_VALUE_LENGTH;
140   // TODO : should we make this configurable
141   private static final int TAG_LENGTH = 256;
142   private static final DecimalFormat FMT = new DecimalFormat("0.##");
143   private static final MathContext CXT = MathContext.DECIMAL64;
144   private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
145   private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
146   private static final TestOptions DEFAULT_OPTS = new TestOptions();
147 
148   private static Map<String, CmdDescriptor> COMMANDS = new TreeMap<String, CmdDescriptor>();
149   private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
150 
151   static {
152     addCommandDescriptor(RandomReadTest.class, "randomRead",
153       "Run random read test");
154     addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan",
155       "Run random seek and scan 100 test");
156     addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",
157       "Run random seek scan with both start and stop row (max 10 rows)");
158     addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100",
159       "Run random seek scan with both start and stop row (max 100 rows)");
160     addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000",
161       "Run random seek scan with both start and stop row (max 1000 rows)");
162     addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000",
163       "Run random seek scan with both start and stop row (max 10000 rows)");
164     addCommandDescriptor(RandomWriteTest.class, "randomWrite",
165       "Run random write test");
166     addCommandDescriptor(SequentialReadTest.class, "sequentialRead",
167       "Run sequential read test");
168     addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",
169       "Run sequential write test");
170     addCommandDescriptor(ScanTest.class, "scan",
171       "Run scan test (read every row)");
172     addCommandDescriptor(FilteredScanTest.class, "filterScan",
173       "Run scan test using a filter to find a specific row based on it's value " +
174       "(make sure to use --rows=20)");
175     addCommandDescriptor(IncrementTest.class, "increment",
176       "Increment on each row; clients overlap on keyspace so some concurrent operations");
177     addCommandDescriptor(AppendTest.class, "append",
178       "Append on each row; clients overlap on keyspace so some concurrent operations");
179     addCommandDescriptor(CheckAndMutateTest.class, "checkAndMutate",
180       "CheckAndMutate on each row; clients overlap on keyspace so some concurrent operations");
181     addCommandDescriptor(CheckAndPutTest.class, "checkAndPut",
182       "CheckAndPut on each row; clients overlap on keyspace so some concurrent operations");
183     addCommandDescriptor(CheckAndDeleteTest.class, "checkAndDelete",
184       "CheckAndDelete on each row; clients overlap on keyspace so some concurrent operations");
185   }
186 
187   /**
188    * Enum for map metrics.  Keep it out here rather than inside in the Map
189    * inner-class so we can find associated properties.
190    */
191   protected static enum Counter {
192     /** elapsed time */
193     ELAPSED_TIME,
194     /** number of rows */
195     ROWS
196   }
197 
198   protected static class RunResult implements Comparable<RunResult> {
199     public RunResult(long duration, Histogram hist) {
200       this.duration = duration;
201       this.hist = hist;
202     }
203 
204     public final long duration;
205     public final Histogram hist;
206 
207     @Override
208     public String toString() {
209       return Long.toString(duration);
210     }
211 
212     @Override public int compareTo(RunResult o) {
213       return Long.compare(this.duration, o.duration);
214     }
215   }
216 
217   /**
218    * Constructor
219    * @param conf Configuration object
220    */
221   public PerformanceEvaluation(final Configuration conf) {
222     super(conf);
223   }
224 
225   protected static void addCommandDescriptor(Class<? extends Test> cmdClass,
226       String name, String description) {
227     CmdDescriptor cmdDescriptor = new CmdDescriptor(cmdClass, name, description);
228     COMMANDS.put(name, cmdDescriptor);
229   }
230 
231   /**
232    * Implementations can have their status set.
233    */
234   interface Status {
235     /**
236      * Sets status
237      * @param msg status message
238      * @throws IOException
239      */
240     void setStatus(final String msg) throws IOException;
241   }
242 
243   /**
244    * MapReduce job that runs a performance evaluation client in each map task.
245    */
246   public static class EvaluationMapTask
247       extends Mapper<LongWritable, Text, LongWritable, LongWritable> {
248 
249     /** configuration parameter name that contains the command */
250     public final static String CMD_KEY = "EvaluationMapTask.command";
251     /** configuration parameter name that contains the PE impl */
252     public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl";
253 
254     private Class<? extends Test> cmd;
255 
256     @Override
257     protected void setup(Context context) throws IOException, InterruptedException {
258       this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class);
259 
260       // this is required so that extensions of PE are instantiated within the
261       // map reduce task...
262       Class<? extends PerformanceEvaluation> peClass =
263           forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class);
264       try {
265         peClass.getConstructor(Configuration.class).newInstance(context.getConfiguration());
266       } catch (Exception e) {
267         throw new IllegalStateException("Could not instantiate PE instance", e);
268       }
269     }
270 
271     private <Type> Class<? extends Type> forName(String className, Class<Type> type) {
272       try {
273         return Class.forName(className).asSubclass(type);
274       } catch (ClassNotFoundException e) {
275         throw new IllegalStateException("Could not find class for name: " + className, e);
276       }
277     }
278 
279     @Override
280     protected void map(LongWritable key, Text value, final Context context)
281            throws IOException, InterruptedException {
282 
283       Status status = new Status() {
284         @Override
285         public void setStatus(String msg) {
286            context.setStatus(msg);
287         }
288       };
289 
290       TestOptions opts = GSON.fromJson(value.toString(), TestOptions.class);
291       Configuration conf = HBaseConfiguration.create(context.getConfiguration());
292       final Connection con = ConnectionFactory.createConnection(conf);
293 
294       // Evaluation task
295       RunResult result = PerformanceEvaluation.runOneClient(this.cmd, conf, con, opts, status);
296       // Collect how much time the thing took. Report as map output and
297       // to the ELAPSED_TIME counter.
298       context.getCounter(Counter.ELAPSED_TIME).increment(result.duration);
299       context.getCounter(Counter.ROWS).increment(opts.perClientRunRows);
300       context.write(new LongWritable(opts.startRow), new LongWritable(result.duration));
301       context.progress();
302     }
303   }
304 
305   /*
306    * If table does not already exist, create. Also create a table when
307    * {@code opts.presplitRegions} is specified or when the existing table's
308    * region replica count doesn't match {@code opts.replicas}.
309    */
310   static boolean checkTable(Admin admin, TestOptions opts) throws IOException {
311     TableName tableName = TableName.valueOf(opts.tableName);
312     boolean needsDelete = false, exists = admin.tableExists(tableName);
313     boolean isReadCmd = opts.cmdName.toLowerCase(Locale.ROOT).contains("read")
314       || opts.cmdName.toLowerCase(Locale.ROOT).contains("scan");
315     if (!exists && isReadCmd) {
316       throw new IllegalStateException(
317         "Must specify an existing table for read commands. Run a write command first.");
318     }
319     HTableDescriptor desc =
320       exists ? admin.getTableDescriptor(TableName.valueOf(opts.tableName)) : null;
321     byte[][] splits = getSplits(opts);
322 
323     // recreate the table when user has requested presplit or when existing
324     // {RegionSplitPolicy,replica count} does not match requested.
325     if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
326       || (!isReadCmd && desc != null &&
327           !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy))
328       || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)) {
329       needsDelete = true;
330       // wait, why did it delete my table?!?
331       LOG.debug(Objects.toStringHelper("needsDelete")
332         .add("needsDelete", needsDelete)
333         .add("isReadCmd", isReadCmd)
334         .add("exists", exists)
335         .add("desc", desc)
336         .add("presplit", opts.presplitRegions)
337         .add("splitPolicy", opts.splitPolicy)
338         .add("replicas", opts.replicas));
339     }
340 
341     // remove an existing table
342     if (needsDelete) {
343       if (admin.isTableEnabled(tableName)) {
344         admin.disableTable(tableName);
345       }
346       admin.deleteTable(tableName);
347     }
348 
349     // table creation is necessary
350     if (!exists || needsDelete) {
351       desc = getTableDescriptor(opts);
352       if (splits != null) {
353         if (LOG.isDebugEnabled()) {
354           for (int i = 0; i < splits.length; i++) {
355             LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i]));
356           }
357         }
358       }
359       admin.createTable(desc, splits);
360       LOG.info("Table " + desc + " created");
361     }
362     return admin.tableExists(tableName);
363   }
364 
365   /**
366    * Create an HTableDescriptor from provided TestOptions.
367    */
368   protected static HTableDescriptor getTableDescriptor(TestOptions opts) {
369     HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName));
370     for (int family = 0; family < opts.families; family++) {
371       byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
372       HColumnDescriptor familyDesc = new HColumnDescriptor(familyName);
373       familyDesc.setDataBlockEncoding(opts.blockEncoding);
374       familyDesc.setCompressionType(opts.compression);
375       familyDesc.setBloomFilterType(opts.bloomType);
376       familyDesc.setBlocksize(opts.blockSize);
377       if (opts.inMemoryCF) {
378         familyDesc.setInMemory(true);
379       }
380       tableDesc.addFamily(familyDesc);
381     }
382     if (opts.replicas != DEFAULT_OPTS.replicas) {
383       tableDesc.setRegionReplication(opts.replicas);
384     }
385     if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) {
386       tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy);
387     }
388     return tableDesc;
389   }
390 
391   /**
392    * generates splits based on total number of rows and specified split regions
393    */
394   protected static byte[][] getSplits(TestOptions opts) {
395     if (opts.presplitRegions == DEFAULT_OPTS.presplitRegions)
396       return null;
397 
398     int numSplitPoints = opts.presplitRegions - 1;
399     byte[][] splits = new byte[numSplitPoints][];
400     int jump = opts.totalRows / opts.presplitRegions;
401     for (int i = 0; i < numSplitPoints; i++) {
402       int rowkey = jump * (1 + i);
403       splits[i] = format(rowkey);
404     }
405     return splits;
406   }
407 
408   /*
409    * Run all clients in this vm each to its own thread.
410    */
411   static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf)
412       throws IOException, InterruptedException {
413     final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
414     assert cmd != null;
415     @SuppressWarnings("unchecked")
416     Future<RunResult>[] threads = new Future[opts.numClientThreads];
417     RunResult[] results = new RunResult[opts.numClientThreads];
418     ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
419       new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
420     final Connection con = ConnectionFactory.createConnection(conf);
421     for (int i = 0; i < threads.length; i++) {
422       final int index = i;
423       threads[i] = pool.submit(new Callable<RunResult>() {
424         @Override
425         public RunResult call() throws Exception {
426           TestOptions threadOpts = new TestOptions(opts);
427           if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;
428           RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() {
429             @Override
430             public void setStatus(final String msg) throws IOException {
431               LOG.info(msg);
432             }
433           });
434           LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration +
435             "ms over " + threadOpts.perClientRunRows + " rows");
436           return run;
437         }
438       });
439     }
440     pool.shutdown();
441 
442     for (int i = 0; i < threads.length; i++) {
443       try {
444         results[i] = threads[i].get();
445       } catch (ExecutionException e) {
446         throw new IOException(e.getCause());
447       }
448     }
449     final String test = cmd.getSimpleName();
450     LOG.info("[" + test + "] Summary of timings (ms): "
451              + Arrays.toString(results));
452     Arrays.sort(results);
453     long total = 0;
454     for (RunResult result : results) {
455       total += result.duration;
456     }
457     LOG.info("[" + test + "]"
458       + "\tMin: " + results[0] + "ms"
459       + "\tMax: " + results[results.length - 1] + "ms"
460       + "\tAvg: " + (total / results.length) + "ms");
461 
462     con.close();
463 
464     return results;
465   }
466 
467   /*
468    * Run a mapreduce job.  Run as many maps as asked-for clients.
469    * Before we start up the job, write out an input file with instruction
470    * per client regards which row they are to start on.
471    * @param cmd Command to run.
472    * @throws IOException
473    */
474   static Job doMapReduce(TestOptions opts, final Configuration conf)
475       throws IOException, InterruptedException, ClassNotFoundException {
476     final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
477     assert cmd != null;
478     Path inputDir = writeInputFile(conf, opts);
479     conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
480     conf.set(EvaluationMapTask.PE_KEY, PerformanceEvaluation.class.getName());
481     Job job = Job.getInstance(conf);
482     job.setJarByClass(PerformanceEvaluation.class);
483     job.setJobName("HBase Performance Evaluation - " + opts.cmdName);
484 
485     job.setInputFormatClass(NLineInputFormat.class);
486     NLineInputFormat.setInputPaths(job, inputDir);
487     // this is default, but be explicit about it just in case.
488     NLineInputFormat.setNumLinesPerSplit(job, 1);
489 
490     job.setOutputKeyClass(LongWritable.class);
491     job.setOutputValueClass(LongWritable.class);
492 
493     job.setMapperClass(EvaluationMapTask.class);
494     job.setReducerClass(LongSumReducer.class);
495 
496     job.setNumReduceTasks(1);
497 
498     job.setOutputFormatClass(TextOutputFormat.class);
499     TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
500 
501     TableMapReduceUtil.addDependencyJars(job);
502     TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
503       Histogram.class,     // yammer metrics
504       Gson.class);  // gson
505 
506     TableMapReduceUtil.initCredentials(job);
507 
508     job.waitForCompletion(true);
509     return job;
510   }
511 
512   /*
513    * Write input file of offsets-per-client for the mapreduce job.
514    * @param c Configuration
515    * @return Directory that contains file written.
516    * @throws IOException
517    */
518   private static Path writeInputFile(final Configuration c, final TestOptions opts) throws IOException {
519     SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
520     Path jobdir = new Path(PERF_EVAL_DIR, formatter.format(new Date()));
521     Path inputDir = new Path(jobdir, "inputs");
522 
523     FileSystem fs = FileSystem.get(c);
524     fs.mkdirs(inputDir);
525 
526     Path inputFile = new Path(inputDir, "input.txt");
527     PrintStream out = new PrintStream(fs.create(inputFile));
528     // Make input random.
529     Map<Integer, String> m = new TreeMap<Integer, String>();
530     Hash h = MurmurHash.getInstance();
531     int perClientRows = (opts.totalRows / opts.numClientThreads);
532     try {
533       for (int i = 0; i < 10; i++) {
534         for (int j = 0; j < opts.numClientThreads; j++) {
535           TestOptions next = new TestOptions(opts);
536           next.startRow = (j * perClientRows) + (i * (perClientRows/10));
537           next.perClientRunRows = perClientRows / 10;
538           String s = GSON.toJson(next);
539           LOG.info("maptask input=" + s);
540           int hash = h.hash(Bytes.toBytes(s));
541           m.put(hash, s);
542         }
543       }
544       for (Map.Entry<Integer, String> e: m.entrySet()) {
545         out.println(e.getValue());
546       }
547     } finally {
548       out.close();
549     }
550     return inputDir;
551   }
552 
553   /**
554    * Describes a command.
555    */
556   static class CmdDescriptor {
557     private Class<? extends Test> cmdClass;
558     private String name;
559     private String description;
560 
561     CmdDescriptor(Class<? extends Test> cmdClass, String name, String description) {
562       this.cmdClass = cmdClass;
563       this.name = name;
564       this.description = description;
565     }
566 
567     public Class<? extends Test> getCmdClass() {
568       return cmdClass;
569     }
570 
571     public String getName() {
572       return name;
573     }
574 
575     public String getDescription() {
576       return description;
577     }
578   }
579 
580   /**
581    * Wraps up options passed to {@link org.apache.hadoop.hbase.PerformanceEvaluation}.
582    * This makes tracking all these arguments a little easier.
583    * NOTE: ADDING AN OPTION, you need to add a data member, a getter/setter (to make JSON
584    * serialization of this TestOptions class behave), and you need to add to the clone constructor
585    * below copying your new option from the 'that' to the 'this'.  Look for 'clone' below.
586    */
587   static class TestOptions {
588     String cmdName = null;
589     boolean nomapred = false;
590     boolean filterAll = false;
591     int startRow = 0;
592     float size = 1.0f;
593     int perClientRunRows = DEFAULT_ROWS_PER_GB;
594     int numClientThreads = 1;
595     int totalRows = DEFAULT_ROWS_PER_GB;
596     float sampleRate = 1.0f;
597     double traceRate = 0.0;
598     String tableName = TABLE_NAME;
599     boolean flushCommits = true;
600     boolean writeToWAL = true;
601     boolean autoFlush = false;
602     boolean oneCon = false;
603     boolean useTags = false;
604     int noOfTags = 1;
605     boolean reportLatency = false;
606     int multiGet = 0;
607     int randomSleep = 0;
608     boolean inMemoryCF = false;
609     int presplitRegions = 0;
610     int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;
611     String splitPolicy = null;
612     Compression.Algorithm compression = Compression.Algorithm.NONE;
613     BloomType bloomType = BloomType.ROW;
614     int blockSize = HConstants.DEFAULT_BLOCKSIZE;
615     DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
616     boolean valueRandom = false;
617     boolean valueZipf = false;
618     int valueSize = DEFAULT_VALUE_LENGTH;
619     int period = (this.perClientRunRows / 10) == 0? perClientRunRows: perClientRunRows / 10;
620     int columns = 1;
621     int families = 1;
622     int caching = 30;
623     boolean addColumns = true;
624     long bufferSize = 2l * 1024l * 1024l;
625 
626     public TestOptions() {}
627 
628     /**
629      * Clone constructor.
630      * @param that Object to copy from.
631      */
632     public TestOptions(TestOptions that) {
633       this.cmdName = that.cmdName;
634       this.nomapred = that.nomapred;
635       this.startRow = that.startRow;
636       this.size = that.size;
637       this.perClientRunRows = that.perClientRunRows;
638       this.numClientThreads = that.numClientThreads;
639       this.totalRows = that.totalRows;
640       this.sampleRate = that.sampleRate;
641       this.traceRate = that.traceRate;
642       this.tableName = that.tableName;
643       this.flushCommits = that.flushCommits;
644       this.writeToWAL = that.writeToWAL;
645       this.autoFlush = that.autoFlush;
646       this.oneCon = that.oneCon;
647       this.useTags = that.useTags;
648       this.noOfTags = that.noOfTags;
649       this.reportLatency = that.reportLatency;
650       this.multiGet = that.multiGet;
651       this.inMemoryCF = that.inMemoryCF;
652       this.presplitRegions = that.presplitRegions;
653       this.replicas = that.replicas;
654       this.splitPolicy = that.splitPolicy;
655       this.compression = that.compression;
656       this.blockEncoding = that.blockEncoding;
657       this.filterAll = that.filterAll;
658       this.bloomType = that.bloomType;
659       this.blockSize = that.blockSize;
660       this.valueRandom = that.valueRandom;
661       this.valueZipf = that.valueZipf;
662       this.valueSize = that.valueSize;
663       this.period = that.period;
664       this.randomSleep = that.randomSleep;
665       this.addColumns = that.addColumns;
666       this.columns = that.columns;
667       this.families = that.families;
668       this.caching = that.caching;
669       this.bufferSize = that.bufferSize;
670     }
671 
672     public int getCaching() {
673       return this.caching;
674     }
675 
676     public void setCaching(final int caching) {
677       this.caching = caching;
678     }
679 
680     public int getColumns() {
681       return this.columns;
682     }
683 
684     public void setColumns(final int columns) {
685       this.columns = columns;
686     }
687 
688     public int getFamilies() {
689       return this.families;
690     }
691 
692     public void setFamilies(final int families) {
693       this.families = families;
694     }
695 
696     public boolean isValueZipf() {
697       return valueZipf;
698     }
699 
700     public void setValueZipf(boolean valueZipf) {
701       this.valueZipf = valueZipf;
702     }
703 
704     public String getCmdName() {
705       return cmdName;
706     }
707 
708     public void setCmdName(String cmdName) {
709       this.cmdName = cmdName;
710     }
711 
712     public int getRandomSleep() {
713       return randomSleep;
714     }
715 
716     public void setRandomSleep(int randomSleep) {
717       this.randomSleep = randomSleep;
718     }
719 
720     public int getReplicas() {
721       return replicas;
722     }
723 
724     public void setReplicas(int replicas) {
725       this.replicas = replicas;
726     }
727 
728     public String getSplitPolicy() {
729       return splitPolicy;
730     }
731 
732     public void setSplitPolicy(String splitPolicy) {
733       this.splitPolicy = splitPolicy;
734     }
735 
736     public void setNomapred(boolean nomapred) {
737       this.nomapred = nomapred;
738     }
739 
740     public void setFilterAll(boolean filterAll) {
741       this.filterAll = filterAll;
742     }
743 
744     public void setStartRow(int startRow) {
745       this.startRow = startRow;
746     }
747 
748     public void setSize(float size) {
749       this.size = size;
750     }
751 
752     public void setPerClientRunRows(int perClientRunRows) {
753       this.perClientRunRows = perClientRunRows;
754     }
755 
756     public void setNumClientThreads(int numClientThreads) {
757       this.numClientThreads = numClientThreads;
758     }
759 
760     public void setTotalRows(int totalRows) {
761       this.totalRows = totalRows;
762     }
763 
764     public void setSampleRate(float sampleRate) {
765       this.sampleRate = sampleRate;
766     }
767 
768     public void setTraceRate(double traceRate) {
769       this.traceRate = traceRate;
770     }
771 
772     public void setTableName(String tableName) {
773       this.tableName = tableName;
774     }
775 
776     public void setFlushCommits(boolean flushCommits) {
777       this.flushCommits = flushCommits;
778     }
779 
780     public void setWriteToWAL(boolean writeToWAL) {
781       this.writeToWAL = writeToWAL;
782     }
783 
784     public void setAutoFlush(boolean autoFlush) {
785       this.autoFlush = autoFlush;
786     }
787 
788     public void setOneCon(boolean oneCon) {
789       this.oneCon = oneCon;
790     }
791 
792     public void setUseTags(boolean useTags) {
793       this.useTags = useTags;
794     }
795 
796     public void setNoOfTags(int noOfTags) {
797       this.noOfTags = noOfTags;
798     }
799 
800     public void setReportLatency(boolean reportLatency) {
801       this.reportLatency = reportLatency;
802     }
803 
804     public void setMultiGet(int multiGet) {
805       this.multiGet = multiGet;
806     }
807 
808     public void setInMemoryCF(boolean inMemoryCF) {
809       this.inMemoryCF = inMemoryCF;
810     }
811 
812     public void setPresplitRegions(int presplitRegions) {
813       this.presplitRegions = presplitRegions;
814     }
815 
816     public void setCompression(Compression.Algorithm compression) {
817       this.compression = compression;
818     }
819 
820     public void setBloomType(BloomType bloomType) {
821       this.bloomType = bloomType;
822     }
823 
824     public void setBlockSize(int blockSize) {
825       this.blockSize = blockSize;
826     }
827 
828     public void setBlockEncoding(DataBlockEncoding blockEncoding) {
829       this.blockEncoding = blockEncoding;
830     }
831 
832     public void setValueRandom(boolean valueRandom) {
833       this.valueRandom = valueRandom;
834     }
835 
836     public void setValueSize(int valueSize) {
837       this.valueSize = valueSize;
838     }
839 
840     public void setBufferSize(long bufferSize) {
841       this.bufferSize = bufferSize;
842     }
843 
844     public long getBufferSize() {
845       return this.bufferSize;
846     }
847 
848     public void setPeriod(int period) {
849       this.period = period;
850     }
851 
852     public boolean isNomapred() {
853       return nomapred;
854     }
855 
856     public boolean isFilterAll() {
857       return filterAll;
858     }
859 
860     public int getStartRow() {
861       return startRow;
862     }
863 
864     public float getSize() {
865       return size;
866     }
867 
868     public int getPerClientRunRows() {
869       return perClientRunRows;
870     }
871 
872     public int getNumClientThreads() {
873       return numClientThreads;
874     }
875 
876     public int getTotalRows() {
877       return totalRows;
878     }
879 
880     public float getSampleRate() {
881       return sampleRate;
882     }
883 
884     public double getTraceRate() {
885       return traceRate;
886     }
887 
888     public String getTableName() {
889       return tableName;
890     }
891 
892     public boolean isFlushCommits() {
893       return flushCommits;
894     }
895 
896     public boolean isWriteToWAL() {
897       return writeToWAL;
898     }
899 
900     public boolean isAutoFlush() {
901       return autoFlush;
902     }
903 
904     public boolean isUseTags() {
905       return useTags;
906     }
907 
908     public int getNoOfTags() {
909       return noOfTags;
910     }
911 
912     public boolean isReportLatency() {
913       return reportLatency;
914     }
915 
916     public int getMultiGet() {
917       return multiGet;
918     }
919 
920     public boolean isInMemoryCF() {
921       return inMemoryCF;
922     }
923 
924     public int getPresplitRegions() {
925       return presplitRegions;
926     }
927 
928     public Compression.Algorithm getCompression() {
929       return compression;
930     }
931 
932     public DataBlockEncoding getBlockEncoding() {
933       return blockEncoding;
934     }
935 
936     public boolean isValueRandom() {
937       return valueRandom;
938     }
939 
940     public int getValueSize() {
941       return valueSize;
942     }
943 
944     public int getPeriod() {
945       return period;
946     }
947 
948     public BloomType getBloomType() {
949       return bloomType;
950     }
951 
952     public int getBlockSize() {
953       return blockSize;
954     }
955 
956     public boolean isOneCon() {
957       return oneCon;
958     }
959 
960     public boolean getAddColumns() {
961       return addColumns;
962     }
963 
964     public void setAddColumns(boolean addColumns) {
965       this.addColumns = addColumns;
966     }
967   }
968 
969   /*
970    * A test.
971    * Subclass to particularize what happens per row.
972    */
973   static abstract class Test {
974     // Below is make it so when Tests are all running in the one
975     // jvm, that they each have a differently seeded Random.
976     private static final Random randomSeed = new Random(System.currentTimeMillis());
977 
978     private static long nextRandomSeed() {
979       return randomSeed.nextLong();
980     }
981     private final int everyN;
982 
983     protected final Random rand = new Random(nextRandomSeed());
984     protected final Configuration conf;
985     protected final TestOptions opts;
986 
987     private final Status status;
988     private final Sampler<?> traceSampler;
989     private final SpanReceiverHost receiverHost;
990     protected Connection connection;
991 
992     private String testName;
993     private Histogram latencyHistogram;
994     private Histogram valueSizeHistogram;
995     private Histogram rpcCallsHistogram;
996     private Histogram remoteRpcCallsHistogram;
997     private Histogram millisBetweenNextHistogram;
998     private Histogram regionsScannedHistogram;
999     private Histogram bytesInResultsHistogram;
1000     private Histogram bytesInRemoteResultsHistogram;
1001     private RandomDistribution.Zipf zipf;
1002 
1003     /**
1004      * Note that all subclasses of this class must provide a public constructor
1005      * that has the exact same list of arguments.
1006      */
1007     Test(final Connection con, final TestOptions options, final Status status) {
1008       this.connection = con;
1009       this.conf = con == null ? HBaseConfiguration.create() : this.connection.getConfiguration();
1010       this.opts = options;
1011       this.status = status;
1012       this.testName = this.getClass().getSimpleName();
1013       receiverHost = SpanReceiverHost.getInstance(conf);
1014       if (options.traceRate >= 1.0) {
1015         this.traceSampler = Sampler.ALWAYS;
1016       } else if (options.traceRate > 0.0) {
1017         conf.setDouble("hbase.sampler.fraction", options.traceRate);
1018         this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
1019       } else {
1020         this.traceSampler = Sampler.NEVER;
1021       }
1022       everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
1023       if (options.isValueZipf()) {
1024         this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.1);
1025       }
1026       LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
1027     }
1028 
1029     int getValueLength(final Random r) {
1030       if (this.opts.isValueRandom()) {
1031         return r.nextInt(opts.valueSize);
1032       } else if (this.opts.isValueZipf()) {
1033         return Math.abs(this.zipf.nextInt());
1034       } else {
1035         return opts.valueSize;
1036       }
1037     }
1038 
1039     void updateValueSize(final Result [] rs) throws IOException {
1040       if (rs == null || !isRandomValueSize()) return;
1041       for (Result r: rs) updateValueSize(r);
1042     }
1043 
1044     void updateValueSize(final Result r) throws IOException {
1045       if (r == null || !isRandomValueSize()) return;
1046       int size = 0;
1047       for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
1048         size += scanner.current().getValueLength();
1049       }
1050       updateValueSize(size);
1051     }
1052 
1053     void updateValueSize(final int valueSize) {
1054       if (!isRandomValueSize()) return;
1055       this.valueSizeHistogram.update(valueSize);
1056     }
1057 
1058     void updateScanMetrics(final ScanMetrics metrics) {
1059       if (metrics == null) return;
1060       Map<String,Long> metricsMap = metrics.getMetricsMap();
1061       Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
1062       if (rpcCalls != null) {
1063         this.rpcCallsHistogram.update(rpcCalls.longValue());
1064       }
1065       Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
1066       if (remoteRpcCalls != null) {
1067         this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
1068       }
1069       Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
1070       if (millisBetweenNext != null) {
1071         this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
1072       }
1073       Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
1074       if (regionsScanned != null) {
1075         this.regionsScannedHistogram.update(regionsScanned.longValue());
1076       }
1077       Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
1078       if (bytesInResults != null && bytesInResults.longValue() > 0) {
1079         this.bytesInResultsHistogram.update(bytesInResults.longValue());
1080       }
1081       Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
1082       if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
1083         this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
1084       }
1085     }
1086 
1087     String generateStatus(final int sr, final int i, final int lr) {
1088       return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
1089         (!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
1090     }
1091 
1092     boolean isRandomValueSize() {
1093       return opts.valueRandom;
1094     }
1095 
1096     protected int getReportingPeriod() {
1097       return opts.period;
1098     }
1099 
1100     /**
1101      * Populated by testTakedown. Only implemented by RandomReadTest at the moment.
1102      */
1103     public Histogram getLatencyHistogram() {
1104       return latencyHistogram;
1105     }
1106 
1107     void testSetup() throws IOException {
1108       // test metrics
1109       latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1110       valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1111       // scan metrics
1112       rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1113       remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1114       millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1115       regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1116       bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1117       bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1118 
1119       if (!opts.oneCon) {
1120         this.connection = ConnectionFactory.createConnection(conf);
1121       }
1122 
1123       onStartup();
1124     }
1125 
1126     abstract void onStartup() throws IOException;
1127 
1128     void testTakedown() throws IOException {
1129       onTakedown();
1130       // Print all stats for this thread continuously.
1131       // Synchronize on Test.class so different threads don't intermingle the
1132       // output. We can't use 'this' here because each thread has its own instance of Test class.
1133       synchronized (Test.class) {
1134         status.setStatus("Test : " + testName + ", Thread : " + Thread.currentThread().getName());
1135         status.setStatus("Latency (us) : " + YammerHistogramUtils.getHistogramReport(
1136             latencyHistogram));
1137         status.setStatus("Num measures (latency) : " + latencyHistogram.count());
1138         status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(latencyHistogram));
1139         status.setStatus("ValueSize (bytes) : "
1140             + YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
1141         status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count());
1142         status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
1143         if (rpcCallsHistogram.count() > 0) {
1144           status.setStatus("rpcCalls (count): " +
1145             YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
1146         }
1147         if (remoteRpcCallsHistogram.count() > 0) {
1148           status.setStatus("remoteRpcCalls (count): " +
1149             YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
1150         }
1151         if (millisBetweenNextHistogram.count() > 0) {
1152           status.setStatus("millisBetweenNext (latency): " +
1153             YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
1154         }
1155         if (regionsScannedHistogram.count() > 0) {
1156           status.setStatus("regionsScanned (count): " +
1157             YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
1158         }
1159         if (bytesInResultsHistogram.count() > 0) {
1160           status.setStatus("bytesInResults (size): " +
1161             YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
1162         }
1163         if (bytesInRemoteResultsHistogram.count() > 0) {
1164           status.setStatus("bytesInRemoteResults (size): " +
1165             YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
1166         }
1167       }
1168       if (!opts.oneCon) {
1169         connection.close();
1170       }
1171       receiverHost.closeReceivers();
1172     }
1173 
1174     abstract void onTakedown() throws IOException;
1175 
1176     /*
1177      * Run test
1178      * @return Elapsed time.
1179      * @throws IOException
1180      */
1181     long test() throws IOException, InterruptedException {
1182       testSetup();
1183       LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
1184       final long startTime = System.nanoTime();
1185       try {
1186         testTimed();
1187       } finally {
1188         testTakedown();
1189       }
1190       return (System.nanoTime() - startTime) / 1000000;
1191     }
1192 
1193     int getStartRow() {
1194       return opts.startRow;
1195     }
1196 
1197     int getLastRow() {
1198       return getStartRow() + opts.perClientRunRows;
1199     }
1200 
1201     /**
1202      * Provides an extension point for tests that don't want a per row invocation.
1203      */
1204     void testTimed() throws IOException, InterruptedException {
1205       int startRow = getStartRow();
1206       int lastRow = getLastRow();
1207       // Report on completion of 1/10th of total.
1208       for (int i = startRow; i < lastRow; i++) {
1209         if (i % everyN != 0) continue;
1210         long startTime = System.nanoTime();
1211         TraceScope scope = Trace.startSpan("test row", traceSampler);
1212         try {
1213           testRow(i);
1214         } finally {
1215           scope.close();
1216         }
1217         // If multiget is enabled, say set to 10, testRow() returns immediately first 9 times
1218         // and sends the actual get request in the 10th iteration. We should only set latency
1219         // when actual request is sent because otherwise it turns out to be 0.
1220         if (opts.multiGet == 0 || (i - startRow + 1) % opts.multiGet == 0) {
1221           latencyHistogram.update((System.nanoTime() - startTime) / 1000);
1222         }
1223         if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
1224           status.setStatus(generateStatus(startRow, i, lastRow));
1225         }
1226       }
1227     }
1228 
1229     /**
1230      * @return Subset of the histograms' calculation.
1231      */
1232     public String getShortLatencyReport() {
1233       return YammerHistogramUtils.getShortHistogramReport(this.latencyHistogram);
1234     }
1235 
1236     /**
1237      * @return Subset of the histograms' calculation.
1238      */
1239     public String getShortValueSizeReport() {
1240       return YammerHistogramUtils.getShortHistogramReport(this.valueSizeHistogram);
1241     }
1242 
1243     /*
1244     * Test for individual row.
1245     * @param i Row index.
1246     */
1247     abstract void testRow(final int i) throws IOException, InterruptedException;
1248   }
1249 
1250   static abstract class TableTest extends Test {
1251     protected Table table;
1252 
1253     TableTest(Connection con, TestOptions options, Status status) {
1254       super(con, options, status);
1255     }
1256 
1257     @Override
1258     void onStartup() throws IOException {
1259       this.table = connection.getTable(TableName.valueOf(opts.tableName));
1260     }
1261 
1262     @Override
1263     void onTakedown() throws IOException {
1264       table.close();
1265     }
1266   }
1267 
1268   static abstract class BufferedMutatorTest extends Test {
1269     protected BufferedMutator mutator;
1270 
1271     BufferedMutatorTest(Connection con, TestOptions options, Status status) {
1272       super(con, options, status);
1273     }
1274 
1275     @Override
1276     void onStartup() throws IOException {
1277       BufferedMutatorParams p = new BufferedMutatorParams(TableName.valueOf(opts.tableName));
1278       p.writeBufferSize(opts.bufferSize);
1279       this.mutator = connection.getBufferedMutator(p);
1280     }
1281 
1282     @Override
1283     void onTakedown() throws IOException {
1284       mutator.close();
1285     }
1286   }
1287 
1288   static class RandomSeekScanTest extends TableTest {
1289     RandomSeekScanTest(Connection con, TestOptions options, Status status) {
1290       super(con, options, status);
1291     }
1292 
1293     @Override
1294     void testRow(final int i) throws IOException {
1295       Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
1296       FilterList list = new FilterList();
1297       for (int family = 0; family < opts.families; family++) {
1298         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1299         if (opts.addColumns) {
1300           for (int column = 0; column < opts.columns; column++) {
1301             byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1302             scan.addColumn(familyName, qualifier);
1303           }
1304         } else {
1305           scan.addFamily(familyName);
1306         }
1307       }
1308       if (opts.filterAll) {
1309         list.addFilter(new FilterAllFilter());
1310       }
1311       list.addFilter(new WhileMatchFilter(new PageFilter(120)));
1312       scan.setFilter(list);
1313       scan.setCaching(opts.caching);
1314       scan.setScanMetricsEnabled(true);
1315       ResultScanner s = this.table.getScanner(scan);
1316       try {
1317         for (Result rr; (rr = s.next()) != null;) {
1318           updateValueSize(rr);
1319         }
1320       } finally {
1321         if (s != null) {
1322           updateScanMetrics(scan.getScanMetrics());
1323           s.close();
1324         }
1325       }
1326     }
1327 
1328     @Override
1329     protected int getReportingPeriod() {
1330       int period = opts.perClientRunRows / 100;
1331       return period == 0 ? opts.perClientRunRows : period;
1332     }
1333 
1334   }
1335 
1336   static abstract class RandomScanWithRangeTest extends TableTest {
1337     RandomScanWithRangeTest(Connection con, TestOptions options, Status status) {
1338       super(con, options, status);
1339     }
1340 
1341     @Override
1342     void testRow(final int i) throws IOException {
1343       Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
1344       Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
1345       for (int family = 0; family < opts.families; family++) {
1346         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1347         if (opts.addColumns) {
1348           for (int column = 0; column < opts.columns; column++) {
1349             byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1350             scan.addColumn(familyName, qualifier);
1351           }
1352         } else {
1353           scan.addFamily(familyName);
1354         }
1355       }
1356       if (opts.filterAll) {
1357         scan.setFilter(new FilterAllFilter());
1358       }
1359       scan.setCaching(opts.caching);
1360       scan.setScanMetricsEnabled(true);
1361       Result r = null;
1362       int count = 0;
1363       ResultScanner s = this.table.getScanner(scan);
1364       try {
1365         for (; (r = s.next()) != null;) {
1366           updateValueSize(r);
1367           count++;
1368         }
1369         if (i % 100 == 0) {
1370           LOG.info(String.format("Scan for key range %s - %s returned %s rows",
1371             Bytes.toString(startAndStopRow.getFirst()),
1372             Bytes.toString(startAndStopRow.getSecond()), count));
1373         }
1374       } finally {
1375         if (s != null) {
1376           updateScanMetrics(scan.getScanMetrics());
1377           s.close();
1378         }
1379       }
1380     }
1381 
1382     protected abstract Pair<byte[],byte[]> getStartAndStopRow();
1383 
1384     protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
1385       int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
1386       int stop = start + maxRange;
1387       return new Pair<byte[],byte[]>(format(start), format(stop));
1388     }
1389 
1390     @Override
1391     protected int getReportingPeriod() {
1392       int period = opts.perClientRunRows / 100;
1393       return period == 0? opts.perClientRunRows: period;
1394     }
1395   }
1396 
1397   static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
1398     RandomScanWithRange10Test(Connection con, TestOptions options, Status status) {
1399       super(con, options, status);
1400     }
1401 
1402     @Override
1403     protected Pair<byte[], byte[]> getStartAndStopRow() {
1404       return generateStartAndStopRows(10);
1405     }
1406   }
1407 
1408   static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
1409     RandomScanWithRange100Test(Connection con, TestOptions options, Status status) {
1410       super(con, options, status);
1411     }
1412 
1413     @Override
1414     protected Pair<byte[], byte[]> getStartAndStopRow() {
1415       return generateStartAndStopRows(100);
1416     }
1417   }
1418 
1419   static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
1420     RandomScanWithRange1000Test(Connection con, TestOptions options, Status status) {
1421       super(con, options, status);
1422     }
1423 
1424     @Override
1425     protected Pair<byte[], byte[]> getStartAndStopRow() {
1426       return generateStartAndStopRows(1000);
1427     }
1428   }
1429 
1430   static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
1431     RandomScanWithRange10000Test(Connection con, TestOptions options, Status status) {
1432       super(con, options, status);
1433     }
1434 
1435     @Override
1436     protected Pair<byte[], byte[]> getStartAndStopRow() {
1437       return generateStartAndStopRows(10000);
1438     }
1439   }
1440 
1441   static class RandomReadTest extends TableTest {
1442     private final Consistency consistency;
1443     private ArrayList<Get> gets;
1444     private Random rd = new Random();
1445 
1446     RandomReadTest(Connection con, TestOptions options, Status status) {
1447       super(con, options, status);
1448       consistency = options.replicas == DEFAULT_OPTS.replicas ? null : Consistency.TIMELINE;
1449       if (opts.multiGet > 0) {
1450         LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
1451         this.gets = new ArrayList<Get>(opts.multiGet);
1452       }
1453     }
1454 
1455     @Override
1456     void testRow(final int i) throws IOException, InterruptedException {
1457       if (opts.randomSleep > 0) {
1458         Thread.sleep(rd.nextInt(opts.randomSleep));
1459       }
1460       Get get = new Get(getRandomRow(this.rand, opts.totalRows));
1461       for (int family = 0; family < opts.families; family++) {
1462         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1463         if (opts.addColumns) {
1464           for (int column = 0; column < opts.columns; column++) {
1465             byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1466             get.addColumn(familyName, qualifier);
1467           }
1468         } else {
1469           get.addFamily(familyName);
1470         }
1471       }
1472       if (opts.filterAll) {
1473         get.setFilter(new FilterAllFilter());
1474       }
1475       get.setConsistency(consistency);
1476       if (LOG.isTraceEnabled()) LOG.trace(get.toString());
1477       if (opts.multiGet > 0) {
1478         this.gets.add(get);
1479         if (this.gets.size() == opts.multiGet) {
1480           Result [] rs = this.table.get(this.gets);
1481           updateValueSize(rs);
1482           this.gets.clear();
1483         }
1484       } else {
1485         updateValueSize(this.table.get(get));
1486       }
1487     }
1488 
1489     @Override
1490     protected int getReportingPeriod() {
1491       int period = opts.perClientRunRows / 10;
1492       return period == 0 ? opts.perClientRunRows : period;
1493     }
1494 
1495     @Override
1496     protected void testTakedown() throws IOException {
1497       if (this.gets != null && this.gets.size() > 0) {
1498         this.table.get(gets);
1499         this.gets.clear();
1500       }
1501       super.testTakedown();
1502     }
1503   }
1504 
1505   static class RandomWriteTest extends BufferedMutatorTest {
1506     RandomWriteTest(Connection con, TestOptions options, Status status) {
1507       super(con, options, status);
1508     }
1509 
1510     @Override
1511     void testRow(final int i) throws IOException {
1512       byte[] row = getRandomRow(this.rand, opts.totalRows);
1513       Put put = new Put(row);
1514       for (int family = 0; family < opts.families; family++) {
1515         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1516         for (int column = 0; column < opts.columns; column++) {
1517           byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1518           byte[] value = generateData(this.rand, getValueLength(this.rand));
1519           if (opts.useTags) {
1520             byte[] tag = generateData(this.rand, TAG_LENGTH);
1521             Tag[] tags = new Tag[opts.noOfTags];
1522             for (int n = 0; n < opts.noOfTags; n++) {
1523               Tag t = new Tag((byte) n, tag);
1524               tags[n] = t;
1525             }
1526             KeyValue kv = new KeyValue(row, familyName, qualifier, HConstants.LATEST_TIMESTAMP,
1527               value, tags);
1528             put.add(kv);
1529             updateValueSize(kv.getValueLength());
1530           } else {
1531             put.add(familyName, qualifier, value);
1532             updateValueSize(value.length);
1533           }
1534         }
1535       }
1536       put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1537       mutator.mutate(put);
1538       if (opts.autoFlush) {
1539         mutator.flush();
1540       }
1541     }
1542   }
1543 
1544   static class ScanTest extends TableTest {
1545     private Scan scan;
1546     private ResultScanner testScanner;
1547 
1548     ScanTest(Connection con, TestOptions options, Status status) {
1549       super(con, options, status);
1550     }
1551 
1552     @Override
1553     void testTakedown() throws IOException {
1554       if (this.testScanner != null) {
1555         this.testScanner.close();
1556       }
1557       super.testTakedown();
1558     }
1559 
1560 
1561     @Override
1562     void testRow(final int i) throws IOException {
1563       if (this.testScanner == null) {
1564         scan = new Scan(format(opts.startRow));
1565         for (int family = 0; family < opts.families; family++) {
1566           byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1567           if (opts.addColumns) {
1568             for (int column = 0; column < opts.columns; column++) {
1569               byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1570               scan.addColumn(familyName, qualifier);
1571             }
1572           } else {
1573             scan.addFamily(familyName);
1574           }
1575         }
1576         if (opts.filterAll) {
1577           scan.setFilter(new FilterAllFilter());
1578         }
1579         scan.setCaching(opts.caching);
1580         scan.setScanMetricsEnabled(true);
1581         this.testScanner = table.getScanner(scan);
1582       }
1583       try {
1584         Result r = testScanner.next();
1585         updateValueSize(r);
1586       } finally {
1587         updateScanMetrics(scan.getScanMetrics());
1588       }
1589     }
1590   }
1591 
1592   /**
1593    * Base class for operations that are CAS-like; that read a value and then set it based off what
1594    * they read. In this category is increment, append, checkAndPut, etc.
1595    *
1596    * <p>These operations also want some concurrency going on. Usually when these tests run, they
1597    * operate in their own part of the key range. In CASTest, we will have them all overlap on the
1598    * same key space. We do this with our getStartRow and getLastRow overrides.
1599    */
1600   static abstract class CASTableTest extends TableTest {
1601     private final byte [] qualifier;
1602     CASTableTest(Connection con, TestOptions options, Status status) {
1603       super(con, options, status);
1604       qualifier = Bytes.toBytes(this.getClass().getSimpleName());
1605     }
1606 
1607     byte [] getQualifier() {
1608       return this.qualifier;
1609     }
1610 
1611     @Override
1612     int getStartRow() {
1613       return 0;
1614     }
1615 
1616     @Override
1617     int getLastRow() {
1618       return opts.perClientRunRows;
1619     }
1620   }
1621 
1622   static class IncrementTest extends CASTableTest {
1623     IncrementTest(Connection con, TestOptions options, Status status) {
1624       super(con, options, status);
1625     }
1626 
1627     @Override
1628     void testRow(final int i) throws IOException {
1629       Increment increment = new Increment(format(i));
1630       // unlike checkAndXXX tests, which make most sense to do on a single value,
1631       // if multiple families are specified for an increment test we assume it is
1632       // meant to raise the work factor
1633       for (int family = 0; family < opts.families; family++) {
1634         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1635         increment.addColumn(familyName, getQualifier(), 1l);
1636       }
1637       updateValueSize(this.table.increment(increment));
1638     }
1639   }
1640 
1641   static class AppendTest extends CASTableTest {
1642     AppendTest(Connection con, TestOptions options, Status status) {
1643       super(con, options, status);
1644     }
1645 
1646     @Override
1647     void testRow(final int i) throws IOException {
1648       byte [] bytes = format(i);
1649       Append append = new Append(bytes);
1650       // unlike checkAndXXX tests, which make most sense to do on a single value,
1651       // if multiple families are specified for an increment test we assume it is
1652       // meant to raise the work factor
1653       for (int family = 0; family < opts.families; family++) {
1654         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1655         append.add(familyName, getQualifier(), bytes);
1656       }
1657       updateValueSize(this.table.append(append));
1658     }
1659   }
1660 
1661   static class CheckAndMutateTest extends CASTableTest {
1662     CheckAndMutateTest(Connection con, TestOptions options, Status status) {
1663       super(con, options, status);
1664     }
1665 
1666     @Override
1667     void testRow(final int i) throws IOException {
1668       final byte [] bytes = format(i);
1669       // checkAndXXX tests operate on only a single value
1670       // Put a known value so when we go to check it, it is there.
1671       Put put = new Put(bytes);
1672       put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1673       this.table.put(put);
1674       RowMutations mutations = new RowMutations(bytes);
1675       mutations.add(put);
1676       this.table.checkAndMutate(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes,
1677           mutations);
1678     }
1679   }
1680 
1681   static class CheckAndPutTest extends CASTableTest {
1682     CheckAndPutTest(Connection con, TestOptions options, Status status) {
1683       super(con, options, status);
1684     }
1685 
1686     @Override
1687     void testRow(final int i) throws IOException {
1688       final byte [] bytes = format(i);
1689       // checkAndXXX tests operate on only a single value
1690       // Put a known value so when we go to check it, it is there.
1691       Put put = new Put(bytes);
1692       put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1693       this.table.put(put);
1694       this.table.checkAndPut(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes, put);
1695     }
1696   }
1697 
1698   static class CheckAndDeleteTest extends CASTableTest {
1699     CheckAndDeleteTest(Connection con, TestOptions options, Status status) {
1700       super(con, options, status);
1701     }
1702 
1703     @Override
1704     void testRow(final int i) throws IOException {
1705       final byte [] bytes = format(i);
1706       // checkAndXXX tests operate on only a single value
1707       // Put a known value so when we go to check it, it is there.
1708       Put put = new Put(bytes);
1709       put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1710       this.table.put(put);
1711       Delete delete = new Delete(put.getRow());
1712       delete.addColumn(FAMILY_ZERO, getQualifier());
1713       this.table.checkAndDelete(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes, delete);
1714     }
1715   }
1716 
1717   static class SequentialReadTest extends TableTest {
1718     SequentialReadTest(Connection con, TestOptions options, Status status) {
1719       super(con, options, status);
1720     }
1721 
1722     @Override
1723     void testRow(final int i) throws IOException {
1724       Get get = new Get(format(i));
1725       for (int family = 0; family < opts.families; family++) {
1726         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1727         if (opts.addColumns) {
1728           for (int column = 0; column < opts.columns; column++) {
1729             byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1730             get.addColumn(familyName, qualifier);
1731           }
1732         } else {
1733           get.addFamily(familyName);
1734         }
1735       }
1736       if (opts.filterAll) {
1737         get.setFilter(new FilterAllFilter());
1738       }
1739       updateValueSize(table.get(get));
1740     }
1741   }
1742 
1743   static class SequentialWriteTest extends BufferedMutatorTest {
1744     SequentialWriteTest(Connection con, TestOptions options, Status status) {
1745       super(con, options, status);
1746     }
1747 
1748     @Override
1749     void testRow(final int i) throws IOException {
1750       byte[] row = format(i);
1751       Put put = new Put(row);
1752       for (int family = 0; family < opts.families; family++) {
1753         byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1754         for (int column = 0; column < opts.columns; column++) {
1755           byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1756           byte[] value = generateData(this.rand, getValueLength(this.rand));
1757           if (opts.useTags) {
1758             byte[] tag = generateData(this.rand, TAG_LENGTH);
1759             Tag[] tags = new Tag[opts.noOfTags];
1760             for (int n = 0; n < opts.noOfTags; n++) {
1761               Tag t = new Tag((byte) n, tag);
1762               tags[n] = t;
1763             }
1764             KeyValue kv = new KeyValue(row, familyName, qualifier, HConstants.LATEST_TIMESTAMP,
1765               value, tags);
1766             put.add(kv);
1767             updateValueSize(kv.getValueLength());
1768           } else {
1769             put.add(familyName, qualifier, value);
1770             updateValueSize(value.length);
1771           }
1772         }
1773       }
1774       put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1775       mutator.mutate(put);
1776       if (opts.autoFlush) {
1777         mutator.flush();
1778       }
1779     }
1780   }
1781 
1782   static class FilteredScanTest extends TableTest {
1783     protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
1784 
1785     FilteredScanTest(Connection con, TestOptions options, Status status) {
1786       super(con, options, status);
1787       if (opts.perClientRunRows == DEFAULT_ROWS_PER_GB) {
1788         LOG.warn("Option \"rows\" unspecified. Using default value " + DEFAULT_ROWS_PER_GB
1789             + ". This could take a very long time.");
1790       }
1791     }
1792 
1793     @Override
1794     void testRow(int i) throws IOException {
1795       byte[] value = generateData(this.rand, getValueLength(this.rand));
1796       Scan scan = constructScan(value);
1797       ResultScanner scanner = null;
1798       try {
1799         scanner = this.table.getScanner(scan);
1800         for (Result r = null; (r = scanner.next()) != null;) {
1801           updateValueSize(r);
1802         }
1803       } finally {
1804         if (scanner != null) {
1805           updateScanMetrics(scan.getScanMetrics());
1806           scanner.close();
1807         }
1808       }
1809     }
1810 
1811     protected Scan constructScan(byte[] valuePrefix) throws IOException {
1812       FilterList list = new FilterList();
1813       Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO,
1814         CompareFilter.CompareOp.EQUAL, new BinaryComparator(valuePrefix));
1815       list.addFilter(filter);
1816       if (opts.filterAll) {
1817         list.addFilter(new FilterAllFilter());
1818       }
1819       Scan scan = new Scan();
1820       if (opts.addColumns) {
1821         for (int column = 0; column < opts.columns; column++) {
1822           byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1823           scan.addColumn(FAMILY_ZERO, qualifier);
1824         }
1825       } else {
1826         scan.addFamily(FAMILY_ZERO);
1827       }
1828       scan.setFilter(list);
1829       scan.setCaching(opts.caching);
1830       scan.setScanMetricsEnabled(true);
1831       return scan;
1832     }
1833   }
1834 
1835   /**
1836    * Compute a throughput rate in MB/s.
1837    * @param rows Number of records consumed.
1838    * @param timeMs Time taken in milliseconds.
1839    * @return String value with label, ie '123.76 MB/s'
1840    */
1841   private static String calculateMbps(int rows, long timeMs, final int valueSize, int families,
1842       int columns) {
1843     BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH +
1844       ((valueSize + (FAMILY_NAME_BASE.length()+1) + COLUMN_ZERO.length) * columns) * families);
1845     BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT)
1846       .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT)
1847       .divide(BYTES_PER_MB, CXT);
1848     return FMT.format(mbps) + " MB/s";
1849   }
1850 
1851   /*
1852    * Format passed integer.
1853    * @param number
1854    * @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of passed
1855    * number (Does absolute in case number is negative).
1856    */
1857   public static byte [] format(final int number) {
1858     byte [] b = new byte[ROW_LENGTH];
1859     int d = Math.abs(number);
1860     for (int i = b.length - 1; i >= 0; i--) {
1861       b[i] = (byte)((d % 10) + '0');
1862       d /= 10;
1863     }
1864     return b;
1865   }
1866 
1867   /*
1868    * This method takes some time and is done inline uploading data.  For
1869    * example, doing the mapfile test, generation of the key and value
1870    * consumes about 30% of CPU time.
1871    * @return Generated random value to insert into a table cell.
1872    */
1873   public static byte[] generateData(final Random r, int length) {
1874     byte [] b = new byte [length];
1875     int i;
1876 
1877     for(i = 0; i < (length-8); i += 8) {
1878       b[i] = (byte) (65 + r.nextInt(26));
1879       b[i+1] = b[i];
1880       b[i+2] = b[i];
1881       b[i+3] = b[i];
1882       b[i+4] = b[i];
1883       b[i+5] = b[i];
1884       b[i+6] = b[i];
1885       b[i+7] = b[i];
1886     }
1887 
1888     byte a = (byte) (65 + r.nextInt(26));
1889     for(; i < length; i++) {
1890       b[i] = a;
1891     }
1892     return b;
1893   }
1894 
1895   /**
1896    * @deprecated Use {@link #generateData(java.util.Random, int)} instead.
1897    * @return Generated random value to insert into a table cell.
1898    */
1899   @Deprecated
1900   public static byte[] generateValue(final Random r) {
1901     return generateData(r, DEFAULT_VALUE_LENGTH);
1902   }
1903 
1904   static byte [] getRandomRow(final Random random, final int totalRows) {
1905     return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
1906   }
1907 
1908   static RunResult runOneClient(final Class<? extends Test> cmd, Configuration conf, Connection con,
1909                            TestOptions opts, final Status status)
1910       throws IOException, InterruptedException {
1911     status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
1912       opts.perClientRunRows + " rows");
1913     long totalElapsedTime;
1914 
1915     final Test t;
1916     try {
1917       Constructor<? extends Test> constructor =
1918         cmd.getDeclaredConstructor(Connection.class, TestOptions.class, Status.class);
1919       t = constructor.newInstance(con, opts, status);
1920     } catch (NoSuchMethodException e) {
1921       throw new IllegalArgumentException("Invalid command class: " +
1922           cmd.getName() + ".  It does not provide a constructor as described by " +
1923           "the javadoc comment.  Available constructors are: " +
1924           Arrays.toString(cmd.getConstructors()));
1925     } catch (Exception e) {
1926       throw new IllegalStateException("Failed to construct command class", e);
1927     }
1928     totalElapsedTime = t.test();
1929 
1930     status.setStatus("Finished " + cmd + " in " + totalElapsedTime +
1931       "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
1932       " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
1933           getAverageValueLength(opts), opts.families, opts.columns) + ")");
1934 
1935     return new RunResult(totalElapsedTime, t.getLatencyHistogram());
1936   }
1937 
1938   private static int getAverageValueLength(final TestOptions opts) {
1939     return opts.valueRandom? opts.valueSize/2: opts.valueSize;
1940   }
1941 
1942   private void runTest(final Class<? extends Test> cmd, TestOptions opts) throws IOException,
1943       InterruptedException, ClassNotFoundException {
1944     // Log the configuration we're going to run with. Uses JSON mapper because lazy. It'll do
1945     // the TestOptions introspection for us and dump the output in a readable format.
1946     LOG.info(cmd.getSimpleName() + " test run options=" + GSON.toJson(opts));
1947     try(Connection conn = ConnectionFactory.createConnection(getConf());
1948         Admin admin = conn.getAdmin()) {
1949       checkTable(admin, opts);
1950     }
1951     if (opts.nomapred) {
1952       doLocalClients(opts, getConf());
1953     } else {
1954       doMapReduce(opts, getConf());
1955     }
1956   }
1957 
1958   protected void printUsage() {
1959     printUsage(this.getClass().getName(), null);
1960   }
1961 
1962   protected static void printUsage(final String message) {
1963     printUsage(PerformanceEvaluation.class.getName(), message);
1964   }
1965 
1966   protected static void printUsageAndExit(final String message, final int exitCode) {
1967     printUsage(message);
1968     System.exit(exitCode);
1969   }
1970 
1971   protected static void printUsage(final String className, final String message) {
1972     if (message != null && message.length() > 0) {
1973       System.err.println(message);
1974     }
1975     System.err.println("Usage: java " + className + " \\");
1976     System.err.println("  <OPTIONS> [-D<property=value>]* <command> <nclients>");
1977     System.err.println();
1978     System.err.println("Options:");
1979     System.err.println(" nomapred        Run multiple clients using threads " +
1980         "(rather than use mapreduce)");
1981     System.err.println(" rows            Rows each client runs. Default: " +
1982         DEFAULT_OPTS.getPerClientRunRows());
1983     System.err.println(" size            Total size in GiB. Mutually exclusive with --rows. " +
1984         "Default: 1.0.");
1985     System.err.println(" sampleRate      Execute test on a sample of total " +
1986         "rows. Only supported by randomRead. Default: 1.0");
1987     System.err.println(" traceRate       Enable HTrace spans. Initiate tracing every N rows. " +
1988         "Default: 0");
1989     System.err.println(" table           Alternate table name. Default: 'TestTable'");
1990     System.err.println(" multiGet        If >0, when doing RandomRead, perform multiple gets " +
1991         "instead of single gets. Default: 0");
1992     System.err.println(" compress        Compression type to use (GZ, LZO, ...). Default: 'NONE'");
1993     System.err.println(" flushCommits    Used to determine if the test should flush the table. " +
1994         "Default: false");
1995     System.err.println(" writeToWAL      Set writeToWAL on puts. Default: True");
1996     System.err.println(" autoFlush       Set autoFlush on htable. Default: False");
1997     System.err.println(" oneCon          all the threads share the same connection. Default: False");
1998     System.err.println(" presplit        Create presplit table. If a table with same name exists,"
1999         + " it'll be deleted and recreated (instead of verifying count of its existing regions). "
2000         + "Recommended for accurate perf analysis (see guide). Default: disabled");
2001     System.err.println(" inmemory        Tries to keep the HFiles of the CF " +
2002         "inmemory as far as possible. Not guaranteed that reads are always served " +
2003         "from memory.  Default: false");
2004     System.err.println(" usetags         Writes tags along with KVs. Use with HFile V3. " +
2005         "Default: false");
2006     System.err.println(" numoftags       Specify the no of tags that would be needed. " +
2007         "This works only if usetags is true. Default: " + DEFAULT_OPTS.noOfTags);
2008     System.err.println(" filterAll       Helps to filter out all the rows on the server side" +
2009         " there by not returning any thing back to the client.  Helps to check the server side" +
2010         " performance.  Uses FilterAllFilter internally. ");
2011     System.err.println(" latency         Set to report operation latencies. Default: False");
2012     System.err.println(" bloomFilter      Bloom filter type, one of " + Arrays.toString(BloomType.values()));
2013     System.err.println(" blockEncoding   Block encoding to use. Value should be one of "
2014         + Arrays.toString(DataBlockEncoding.values()) + ". Default: NONE");
2015     System.err.println(" valueSize       Pass value size to use: Default: " +
2016         DEFAULT_OPTS.getValueSize());
2017     System.err.println(" valueRandom     Set if we should vary value size between 0 and " +
2018         "'valueSize'; set on read for stats on size: Default: Not set.");
2019     System.err.println(" valueZipf       Set if we should vary value size between 0 and " +
2020         "'valueSize' in zipf form: Default: Not set.");
2021     System.err.println(" period          Report every 'period' rows: " +
2022         "Default: opts.perClientRunRows / 10 = " + DEFAULT_OPTS.getPerClientRunRows()/10);
2023     System.err.println(" multiGet        Batch gets together into groups of N. Only supported " +
2024         "by randomRead. Default: disabled");
2025     System.err.println(" blockSize       Blocksize to use when writing out hfiles. ");
2026     System.err.println(" addColumns      Adds columns to scans/gets explicitly. Default: true");
2027     System.err.println(" replicas        Enable region replica testing. Defaults: 1.");
2028     System.err.println(" splitPolicy     Specify a custom RegionSplitPolicy for the table.");
2029     System.err.println(" randomSleep     Do a random sleep before each get between 0 and entered value. Defaults: 0");
2030     System.err.println(" columns         Columns to write per row. Default: 1");
2031     System.err.println(" families        Specify number of column families for the table. Default: 1");
2032     System.err.println(" caching         Scan caching to use. Default: 30");
2033     System.err.println(" bufferSize      Set the value of client side buffering. Default: 2MB");
2034     System.err.println();
2035     System.err.println(" Note: -D properties will be applied to the conf used. ");
2036     System.err.println("  For example: ");
2037     System.err.println("   -Dmapreduce.output.fileoutputformat.compress=true");
2038     System.err.println("   -Dmapreduce.task.timeout=60000");
2039     System.err.println();
2040     System.err.println("Command:");
2041     for (CmdDescriptor command : COMMANDS.values()) {
2042       System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
2043     }
2044     System.err.println();
2045     System.err.println("Args:");
2046     System.err.println(" nclients        Integer. Required. Total number of clients "
2047         + "(and HRegionServers) running. 1 <= value <= 500");
2048     System.err.println("Examples:");
2049     System.err.println(" To run a single client doing the default 1M sequentialWrites:");
2050     System.err.println(" $ bin/hbase " + className + " sequentialWrite 1");
2051     System.err.println(" To run 10 clients doing increments over ten rows:");
2052     System.err.println(" $ bin/hbase " + className + " --rows=10 --nomapred increment 10");
2053   }
2054 
2055   /**
2056    * Parse options passed in via an arguments array. Assumes that array has been split
2057    * on white-space and placed into a {@code Queue}. Any unknown arguments will remain
2058    * in the queue at the conclusion of this method call. It's up to the caller to deal
2059    * with these unrecognized arguments.
2060    */
2061   static TestOptions parseOpts(Queue<String> args) {
2062     TestOptions opts = new TestOptions();
2063 
2064     String cmd = null;
2065     while ((cmd = args.poll()) != null) {
2066       if (cmd.equals("-h") || cmd.startsWith("--h")) {
2067         // place item back onto queue so that caller knows parsing was incomplete
2068         args.add(cmd);
2069         break;
2070       }
2071 
2072       final String nmr = "--nomapred";
2073       if (cmd.startsWith(nmr)) {
2074         opts.nomapred = true;
2075         continue;
2076       }
2077 
2078       final String rows = "--rows=";
2079       if (cmd.startsWith(rows)) {
2080         opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
2081         continue;
2082       }
2083 
2084       final String sampleRate = "--sampleRate=";
2085       if (cmd.startsWith(sampleRate)) {
2086         opts.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length()));
2087         continue;
2088       }
2089 
2090       final String table = "--table=";
2091       if (cmd.startsWith(table)) {
2092         opts.tableName = cmd.substring(table.length());
2093         continue;
2094       }
2095 
2096       final String startRow = "--startRow=";
2097       if (cmd.startsWith(startRow)) {
2098         opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
2099         continue;
2100       }
2101 
2102       final String compress = "--compress=";
2103       if (cmd.startsWith(compress)) {
2104         opts.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
2105         continue;
2106       }
2107 
2108       final String traceRate = "--traceRate=";
2109       if (cmd.startsWith(traceRate)) {
2110         opts.traceRate = Double.parseDouble(cmd.substring(traceRate.length()));
2111         continue;
2112       }
2113 
2114       final String blockEncoding = "--blockEncoding=";
2115       if (cmd.startsWith(blockEncoding)) {
2116         opts.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
2117         continue;
2118       }
2119 
2120       final String flushCommits = "--flushCommits=";
2121       if (cmd.startsWith(flushCommits)) {
2122         opts.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
2123         continue;
2124       }
2125 
2126       final String writeToWAL = "--writeToWAL=";
2127       if (cmd.startsWith(writeToWAL)) {
2128         opts.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length()));
2129         continue;
2130       }
2131 
2132       final String presplit = "--presplit=";
2133       if (cmd.startsWith(presplit)) {
2134         opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));
2135         continue;
2136       }
2137 
2138       final String inMemory = "--inmemory=";
2139       if (cmd.startsWith(inMemory)) {
2140         opts.inMemoryCF = Boolean.parseBoolean(cmd.substring(inMemory.length()));
2141         continue;
2142       }
2143 
2144       final String autoFlush = "--autoFlush=";
2145       if (cmd.startsWith(autoFlush)) {
2146         opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length()));
2147         continue;
2148       }
2149 
2150       final String onceCon = "--oneCon=";
2151       if (cmd.startsWith(onceCon)) {
2152         opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length()));
2153         continue;
2154       }
2155 
2156       final String latency = "--latency";
2157       if (cmd.startsWith(latency)) {
2158         opts.reportLatency = true;
2159         continue;
2160       }
2161 
2162       final String multiGet = "--multiGet=";
2163       if (cmd.startsWith(multiGet)) {
2164         opts.multiGet = Integer.parseInt(cmd.substring(multiGet.length()));
2165         continue;
2166       }
2167 
2168       final String useTags = "--usetags=";
2169       if (cmd.startsWith(useTags)) {
2170         opts.useTags = Boolean.parseBoolean(cmd.substring(useTags.length()));
2171         continue;
2172       }
2173 
2174       final String noOfTags = "--numoftags=";
2175       if (cmd.startsWith(noOfTags)) {
2176         opts.noOfTags = Integer.parseInt(cmd.substring(noOfTags.length()));
2177         continue;
2178       }
2179 
2180       final String replicas = "--replicas=";
2181       if (cmd.startsWith(replicas)) {
2182         opts.replicas = Integer.parseInt(cmd.substring(replicas.length()));
2183         continue;
2184       }
2185 
2186       final String filterOutAll = "--filterAll";
2187       if (cmd.startsWith(filterOutAll)) {
2188         opts.filterAll = true;
2189         continue;
2190       }
2191 
2192       final String size = "--size=";
2193       if (cmd.startsWith(size)) {
2194         opts.size = Float.parseFloat(cmd.substring(size.length()));
2195         continue;
2196       }
2197 
2198       final String splitPolicy = "--splitPolicy=";
2199       if (cmd.startsWith(splitPolicy)) {
2200         opts.splitPolicy = cmd.substring(splitPolicy.length());
2201         continue;
2202       }
2203 
2204       final String randomSleep = "--randomSleep=";
2205       if (cmd.startsWith(randomSleep)) {
2206         opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length()));
2207         continue;
2208       }
2209 
2210       final String bloomFilter = "--bloomFilter=";
2211       if (cmd.startsWith(bloomFilter)) {
2212         opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length()));
2213         continue;
2214       }
2215 
2216       final String blockSize = "--blockSize=";
2217       if(cmd.startsWith(blockSize) ) {
2218         opts.blockSize = Integer.parseInt(cmd.substring(blockSize.length()));
2219         continue;
2220       }
2221 
2222       final String valueSize = "--valueSize=";
2223       if (cmd.startsWith(valueSize)) {
2224         opts.valueSize = Integer.parseInt(cmd.substring(valueSize.length()));
2225         continue;
2226       }
2227 
2228       final String valueRandom = "--valueRandom";
2229       if (cmd.startsWith(valueRandom)) {
2230         opts.valueRandom = true;
2231         if (opts.valueZipf) {
2232           throw new IllegalStateException("Either valueZipf or valueRandom but not both");
2233         }
2234         continue;
2235       }
2236 
2237       final String valueZipf = "--valueZipf";
2238       if (cmd.startsWith(valueZipf)) {
2239         opts.valueZipf = true;
2240         if (opts.valueRandom) {
2241           throw new IllegalStateException("Either valueZipf or valueRandom but not both");
2242         }
2243         continue;
2244       }
2245 
2246       final String period = "--period=";
2247       if (cmd.startsWith(period)) {
2248         opts.period = Integer.parseInt(cmd.substring(period.length()));
2249         continue;
2250       }
2251 
2252       final String addColumns = "--addColumns=";
2253       if (cmd.startsWith(addColumns)) {
2254         opts.addColumns = Boolean.parseBoolean(cmd.substring(addColumns.length()));
2255         continue;
2256       }
2257 
2258       final String columns = "--columns=";
2259       if (cmd.startsWith(columns)) {
2260         opts.columns = Integer.parseInt(cmd.substring(columns.length()));
2261         continue;
2262       }
2263 
2264       final String families = "--families=";
2265       if (cmd.startsWith(families)) {
2266         opts.families = Integer.parseInt(cmd.substring(families.length()));
2267         continue;
2268       }
2269 
2270       final String caching = "--caching=";
2271       if (cmd.startsWith(caching)) {
2272         opts.caching = Integer.parseInt(cmd.substring(caching.length()));
2273         continue;
2274       }
2275 
2276       final String bufferSize = "--bufferSize=";
2277       if (cmd.startsWith(bufferSize)) {
2278         opts.bufferSize = Long.parseLong(cmd.substring(bufferSize.length()));
2279         continue;
2280       }
2281 
2282       if (isCommandClass(cmd)) {
2283         opts.cmdName = cmd;
2284         try {
2285           opts.numClientThreads = Integer.parseInt(args.remove());
2286         } catch (NoSuchElementException | NumberFormatException e) {
2287           throw new IllegalArgumentException("Command " + cmd + " does not have threads number", e);
2288         }
2289         int rowsPerGB = getRowsPerGB(opts);
2290         if (opts.size != DEFAULT_OPTS.size &&
2291             opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
2292           throw new IllegalArgumentException(rows + " and " + size + " are mutually exclusive arguments.");
2293         }
2294         if (opts.size != DEFAULT_OPTS.size) {
2295           // total size in GB specified
2296           opts.totalRows = (int) opts.size * rowsPerGB;
2297           opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
2298         } else {
2299           opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
2300           opts.size = opts.totalRows / rowsPerGB;
2301         }
2302         break;
2303       } else {
2304         printUsageAndExit("ERROR: Unrecognized option/command: " + cmd, -1);
2305       }
2306 
2307       // Not matching any option or command.
2308       System.err.println("Error: Wrong option or command: " + cmd);
2309       args.add(cmd);
2310       break;
2311     }
2312     return opts;
2313   }
2314 
2315   static int getRowsPerGB(final TestOptions opts) {
2316     return ONE_GB / ((opts.valueRandom? opts.valueSize/2: opts.valueSize) * opts.getColumns() *
2317         opts.getFamilies());
2318   }
2319 
2320   @Override
2321   public int run(String[] args) throws Exception {
2322     // Process command-line args. TODO: Better cmd-line processing
2323     // (but hopefully something not as painful as cli options).
2324     int errCode = -1;
2325     if (args.length < 1) {
2326       printUsage();
2327       return errCode;
2328     }
2329 
2330     try {
2331       LinkedList<String> argv = new LinkedList<String>();
2332       argv.addAll(Arrays.asList(args));
2333       TestOptions opts = parseOpts(argv);
2334 
2335       // args remaining, print help and exit
2336       if (!argv.isEmpty()) {
2337         errCode = 0;
2338         printUsage();
2339         return errCode;
2340       }
2341 
2342       // must run at least 1 client
2343       if (opts.numClientThreads <= 0) {
2344         throw new IllegalArgumentException("Number of clients must be > 0");
2345       }
2346 
2347       // cmdName should not be null, print help and exit
2348       if (opts.cmdName == null) {
2349         printUsage();
2350         return errCode;
2351       }
2352 
2353       Class<? extends Test> cmdClass = determineCommandClass(opts.cmdName);
2354       if (cmdClass != null) {
2355         runTest(cmdClass, opts);
2356         errCode = 0;
2357       }
2358 
2359     } catch (Exception e) {
2360       e.printStackTrace();
2361     }
2362 
2363     return errCode;
2364   }
2365 
2366   private static boolean isCommandClass(String cmd) {
2367     return COMMANDS.containsKey(cmd);
2368   }
2369 
2370   private static Class<? extends Test> determineCommandClass(String cmd) {
2371     CmdDescriptor descriptor = COMMANDS.get(cmd);
2372     return descriptor != null ? descriptor.getCmdClass() : null;
2373   }
2374 
2375   public static void main(final String[] args) throws Exception {
2376     int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
2377     System.exit(res);
2378   }
2379 }