1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.io.PrintStream;
23 import java.lang.reflect.Constructor;
24 import java.math.BigDecimal;
25 import java.math.MathContext;
26 import java.text.DecimalFormat;
27 import java.text.SimpleDateFormat;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Date;
31 import java.util.LinkedList;
32 import java.util.Locale;
33 import java.util.Map;
34 import java.util.Queue;
35 import java.util.Random;
36 import java.util.TreeMap;
37 import java.util.NoSuchElementException;
38 import java.util.concurrent.Callable;
39 import java.util.concurrent.ExecutionException;
40 import java.util.concurrent.ExecutorService;
41 import java.util.concurrent.Executors;
42 import java.util.concurrent.Future;
43
44 import com.google.common.base.Objects;
45 import com.google.common.util.concurrent.ThreadFactoryBuilder;
46
47 import org.apache.commons.lang.StringUtils;
48 import org.apache.commons.logging.Log;
49 import org.apache.commons.logging.LogFactory;
50 import org.apache.hadoop.conf.Configuration;
51 import org.apache.hadoop.conf.Configured;
52 import org.apache.hadoop.fs.FileSystem;
53 import org.apache.hadoop.fs.Path;
54 import org.apache.hadoop.hbase.classification.InterfaceAudience;
55 import org.apache.hadoop.hbase.client.Admin;
56 import org.apache.hadoop.hbase.client.Append;
57 import org.apache.hadoop.hbase.client.BufferedMutator;
58 import org.apache.hadoop.hbase.client.BufferedMutatorParams;
59 import org.apache.hadoop.hbase.client.Connection;
60 import org.apache.hadoop.hbase.client.ConnectionFactory;
61 import org.apache.hadoop.hbase.client.Consistency;
62 import org.apache.hadoop.hbase.client.Delete;
63 import org.apache.hadoop.hbase.client.Durability;
64 import org.apache.hadoop.hbase.client.Get;
65 import org.apache.hadoop.hbase.client.Increment;
66 import org.apache.hadoop.hbase.client.Put;
67 import org.apache.hadoop.hbase.client.Result;
68 import org.apache.hadoop.hbase.client.ResultScanner;
69 import org.apache.hadoop.hbase.client.RowMutations;
70 import org.apache.hadoop.hbase.client.Scan;
71 import org.apache.hadoop.hbase.client.Table;
72 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
73 import org.apache.hadoop.hbase.filter.BinaryComparator;
74 import org.apache.hadoop.hbase.filter.CompareFilter;
75 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
76 import org.apache.hadoop.hbase.filter.Filter;
77 import org.apache.hadoop.hbase.filter.FilterAllFilter;
78 import org.apache.hadoop.hbase.filter.FilterList;
79 import org.apache.hadoop.hbase.filter.PageFilter;
80 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
81 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
82 import org.apache.hadoop.hbase.io.compress.Compression;
83 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
84 import org.apache.hadoop.hbase.io.hfile.RandomDistribution;
85 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
86 import org.apache.hadoop.hbase.regionserver.BloomType;
87 import org.apache.hadoop.hbase.trace.HBaseHTraceConfiguration;
88 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
89 import org.apache.hadoop.hbase.util.*;
90 import org.apache.hadoop.io.LongWritable;
91 import org.apache.hadoop.io.Text;
92 import org.apache.hadoop.mapreduce.Job;
93 import org.apache.hadoop.mapreduce.Mapper;
94 import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
95 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
96 import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
97 import org.apache.hadoop.util.Tool;
98 import org.apache.hadoop.util.ToolRunner;
99
100 import com.yammer.metrics.core.Histogram;
101 import com.yammer.metrics.stats.UniformSample;
102
103 import org.apache.hbase.thirdparty.com.google.gson.Gson;
104 import org.apache.htrace.Sampler;
105 import org.apache.htrace.Trace;
106 import org.apache.htrace.TraceScope;
107 import org.apache.htrace.impl.ProbabilitySampler;
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
127 public class PerformanceEvaluation extends Configured implements Tool {
128 private static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
129 private static final Gson GSON = GsonUtil.createGson().create();
130
131 public static final String TABLE_NAME = "TestTable";
132 public static final String FAMILY_NAME_BASE = "info";
133 public static final byte[] FAMILY_ZERO = Bytes.toBytes("info0");
134 public static final byte[] COLUMN_ZERO = Bytes.toBytes("" + 0);
135 public static final int DEFAULT_VALUE_LENGTH = 1000;
136 public static final int ROW_LENGTH = 26;
137
138 private static final int ONE_GB = 1024 * 1024 * 1000;
139 private static final int DEFAULT_ROWS_PER_GB = ONE_GB / DEFAULT_VALUE_LENGTH;
140
141 private static final int TAG_LENGTH = 256;
142 private static final DecimalFormat FMT = new DecimalFormat("0.##");
143 private static final MathContext CXT = MathContext.DECIMAL64;
144 private static final BigDecimal MS_PER_SEC = BigDecimal.valueOf(1000);
145 private static final BigDecimal BYTES_PER_MB = BigDecimal.valueOf(1024 * 1024);
146 private static final TestOptions DEFAULT_OPTS = new TestOptions();
147
148 private static Map<String, CmdDescriptor> COMMANDS = new TreeMap<String, CmdDescriptor>();
149 private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
150
151 static {
152 addCommandDescriptor(RandomReadTest.class, "randomRead",
153 "Run random read test");
154 addCommandDescriptor(RandomSeekScanTest.class, "randomSeekScan",
155 "Run random seek and scan 100 test");
156 addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",
157 "Run random seek scan with both start and stop row (max 10 rows)");
158 addCommandDescriptor(RandomScanWithRange100Test.class, "scanRange100",
159 "Run random seek scan with both start and stop row (max 100 rows)");
160 addCommandDescriptor(RandomScanWithRange1000Test.class, "scanRange1000",
161 "Run random seek scan with both start and stop row (max 1000 rows)");
162 addCommandDescriptor(RandomScanWithRange10000Test.class, "scanRange10000",
163 "Run random seek scan with both start and stop row (max 10000 rows)");
164 addCommandDescriptor(RandomWriteTest.class, "randomWrite",
165 "Run random write test");
166 addCommandDescriptor(SequentialReadTest.class, "sequentialRead",
167 "Run sequential read test");
168 addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",
169 "Run sequential write test");
170 addCommandDescriptor(ScanTest.class, "scan",
171 "Run scan test (read every row)");
172 addCommandDescriptor(FilteredScanTest.class, "filterScan",
173 "Run scan test using a filter to find a specific row based on it's value " +
174 "(make sure to use --rows=20)");
175 addCommandDescriptor(IncrementTest.class, "increment",
176 "Increment on each row; clients overlap on keyspace so some concurrent operations");
177 addCommandDescriptor(AppendTest.class, "append",
178 "Append on each row; clients overlap on keyspace so some concurrent operations");
179 addCommandDescriptor(CheckAndMutateTest.class, "checkAndMutate",
180 "CheckAndMutate on each row; clients overlap on keyspace so some concurrent operations");
181 addCommandDescriptor(CheckAndPutTest.class, "checkAndPut",
182 "CheckAndPut on each row; clients overlap on keyspace so some concurrent operations");
183 addCommandDescriptor(CheckAndDeleteTest.class, "checkAndDelete",
184 "CheckAndDelete on each row; clients overlap on keyspace so some concurrent operations");
185 }
186
187
188
189
190
191 protected static enum Counter {
192
193 ELAPSED_TIME,
194
195 ROWS
196 }
197
198 protected static class RunResult implements Comparable<RunResult> {
199 public RunResult(long duration, Histogram hist) {
200 this.duration = duration;
201 this.hist = hist;
202 }
203
204 public final long duration;
205 public final Histogram hist;
206
207 @Override
208 public String toString() {
209 return Long.toString(duration);
210 }
211
212 @Override public int compareTo(RunResult o) {
213 return Long.compare(this.duration, o.duration);
214 }
215 }
216
217
218
219
220
221 public PerformanceEvaluation(final Configuration conf) {
222 super(conf);
223 }
224
225 protected static void addCommandDescriptor(Class<? extends Test> cmdClass,
226 String name, String description) {
227 CmdDescriptor cmdDescriptor = new CmdDescriptor(cmdClass, name, description);
228 COMMANDS.put(name, cmdDescriptor);
229 }
230
231
232
233
234 interface Status {
235
236
237
238
239
240 void setStatus(final String msg) throws IOException;
241 }
242
243
244
245
246 public static class EvaluationMapTask
247 extends Mapper<LongWritable, Text, LongWritable, LongWritable> {
248
249
250 public final static String CMD_KEY = "EvaluationMapTask.command";
251
252 public static final String PE_KEY = "EvaluationMapTask.performanceEvalImpl";
253
254 private Class<? extends Test> cmd;
255
256 @Override
257 protected void setup(Context context) throws IOException, InterruptedException {
258 this.cmd = forName(context.getConfiguration().get(CMD_KEY), Test.class);
259
260
261
262 Class<? extends PerformanceEvaluation> peClass =
263 forName(context.getConfiguration().get(PE_KEY), PerformanceEvaluation.class);
264 try {
265 peClass.getConstructor(Configuration.class).newInstance(context.getConfiguration());
266 } catch (Exception e) {
267 throw new IllegalStateException("Could not instantiate PE instance", e);
268 }
269 }
270
271 private <Type> Class<? extends Type> forName(String className, Class<Type> type) {
272 try {
273 return Class.forName(className).asSubclass(type);
274 } catch (ClassNotFoundException e) {
275 throw new IllegalStateException("Could not find class for name: " + className, e);
276 }
277 }
278
279 @Override
280 protected void map(LongWritable key, Text value, final Context context)
281 throws IOException, InterruptedException {
282
283 Status status = new Status() {
284 @Override
285 public void setStatus(String msg) {
286 context.setStatus(msg);
287 }
288 };
289
290 TestOptions opts = GSON.fromJson(value.toString(), TestOptions.class);
291 Configuration conf = HBaseConfiguration.create(context.getConfiguration());
292 final Connection con = ConnectionFactory.createConnection(conf);
293
294
295 RunResult result = PerformanceEvaluation.runOneClient(this.cmd, conf, con, opts, status);
296
297
298 context.getCounter(Counter.ELAPSED_TIME).increment(result.duration);
299 context.getCounter(Counter.ROWS).increment(opts.perClientRunRows);
300 context.write(new LongWritable(opts.startRow), new LongWritable(result.duration));
301 context.progress();
302 }
303 }
304
305
306
307
308
309
310 static boolean checkTable(Admin admin, TestOptions opts) throws IOException {
311 TableName tableName = TableName.valueOf(opts.tableName);
312 boolean needsDelete = false, exists = admin.tableExists(tableName);
313 boolean isReadCmd = opts.cmdName.toLowerCase(Locale.ROOT).contains("read")
314 || opts.cmdName.toLowerCase(Locale.ROOT).contains("scan");
315 if (!exists && isReadCmd) {
316 throw new IllegalStateException(
317 "Must specify an existing table for read commands. Run a write command first.");
318 }
319 HTableDescriptor desc =
320 exists ? admin.getTableDescriptor(TableName.valueOf(opts.tableName)) : null;
321 byte[][] splits = getSplits(opts);
322
323
324
325 if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
326 || (!isReadCmd && desc != null &&
327 !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy))
328 || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)) {
329 needsDelete = true;
330
331 LOG.debug(Objects.toStringHelper("needsDelete")
332 .add("needsDelete", needsDelete)
333 .add("isReadCmd", isReadCmd)
334 .add("exists", exists)
335 .add("desc", desc)
336 .add("presplit", opts.presplitRegions)
337 .add("splitPolicy", opts.splitPolicy)
338 .add("replicas", opts.replicas));
339 }
340
341
342 if (needsDelete) {
343 if (admin.isTableEnabled(tableName)) {
344 admin.disableTable(tableName);
345 }
346 admin.deleteTable(tableName);
347 }
348
349
350 if (!exists || needsDelete) {
351 desc = getTableDescriptor(opts);
352 if (splits != null) {
353 if (LOG.isDebugEnabled()) {
354 for (int i = 0; i < splits.length; i++) {
355 LOG.debug(" split " + i + ": " + Bytes.toStringBinary(splits[i]));
356 }
357 }
358 }
359 admin.createTable(desc, splits);
360 LOG.info("Table " + desc + " created");
361 }
362 return admin.tableExists(tableName);
363 }
364
365
366
367
368 protected static HTableDescriptor getTableDescriptor(TestOptions opts) {
369 HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName));
370 for (int family = 0; family < opts.families; family++) {
371 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
372 HColumnDescriptor familyDesc = new HColumnDescriptor(familyName);
373 familyDesc.setDataBlockEncoding(opts.blockEncoding);
374 familyDesc.setCompressionType(opts.compression);
375 familyDesc.setBloomFilterType(opts.bloomType);
376 familyDesc.setBlocksize(opts.blockSize);
377 if (opts.inMemoryCF) {
378 familyDesc.setInMemory(true);
379 }
380 tableDesc.addFamily(familyDesc);
381 }
382 if (opts.replicas != DEFAULT_OPTS.replicas) {
383 tableDesc.setRegionReplication(opts.replicas);
384 }
385 if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) {
386 tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy);
387 }
388 return tableDesc;
389 }
390
391
392
393
394 protected static byte[][] getSplits(TestOptions opts) {
395 if (opts.presplitRegions == DEFAULT_OPTS.presplitRegions)
396 return null;
397
398 int numSplitPoints = opts.presplitRegions - 1;
399 byte[][] splits = new byte[numSplitPoints][];
400 int jump = opts.totalRows / opts.presplitRegions;
401 for (int i = 0; i < numSplitPoints; i++) {
402 int rowkey = jump * (1 + i);
403 splits[i] = format(rowkey);
404 }
405 return splits;
406 }
407
408
409
410
411 static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf)
412 throws IOException, InterruptedException {
413 final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
414 assert cmd != null;
415 @SuppressWarnings("unchecked")
416 Future<RunResult>[] threads = new Future[opts.numClientThreads];
417 RunResult[] results = new RunResult[opts.numClientThreads];
418 ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads,
419 new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
420 final Connection con = ConnectionFactory.createConnection(conf);
421 for (int i = 0; i < threads.length; i++) {
422 final int index = i;
423 threads[i] = pool.submit(new Callable<RunResult>() {
424 @Override
425 public RunResult call() throws Exception {
426 TestOptions threadOpts = new TestOptions(opts);
427 if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows;
428 RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() {
429 @Override
430 public void setStatus(final String msg) throws IOException {
431 LOG.info(msg);
432 }
433 });
434 LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration +
435 "ms over " + threadOpts.perClientRunRows + " rows");
436 return run;
437 }
438 });
439 }
440 pool.shutdown();
441
442 for (int i = 0; i < threads.length; i++) {
443 try {
444 results[i] = threads[i].get();
445 } catch (ExecutionException e) {
446 throw new IOException(e.getCause());
447 }
448 }
449 final String test = cmd.getSimpleName();
450 LOG.info("[" + test + "] Summary of timings (ms): "
451 + Arrays.toString(results));
452 Arrays.sort(results);
453 long total = 0;
454 for (RunResult result : results) {
455 total += result.duration;
456 }
457 LOG.info("[" + test + "]"
458 + "\tMin: " + results[0] + "ms"
459 + "\tMax: " + results[results.length - 1] + "ms"
460 + "\tAvg: " + (total / results.length) + "ms");
461
462 con.close();
463
464 return results;
465 }
466
467
468
469
470
471
472
473
474 static Job doMapReduce(TestOptions opts, final Configuration conf)
475 throws IOException, InterruptedException, ClassNotFoundException {
476 final Class<? extends Test> cmd = determineCommandClass(opts.cmdName);
477 assert cmd != null;
478 Path inputDir = writeInputFile(conf, opts);
479 conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
480 conf.set(EvaluationMapTask.PE_KEY, PerformanceEvaluation.class.getName());
481 Job job = Job.getInstance(conf);
482 job.setJarByClass(PerformanceEvaluation.class);
483 job.setJobName("HBase Performance Evaluation - " + opts.cmdName);
484
485 job.setInputFormatClass(NLineInputFormat.class);
486 NLineInputFormat.setInputPaths(job, inputDir);
487
488 NLineInputFormat.setNumLinesPerSplit(job, 1);
489
490 job.setOutputKeyClass(LongWritable.class);
491 job.setOutputValueClass(LongWritable.class);
492
493 job.setMapperClass(EvaluationMapTask.class);
494 job.setReducerClass(LongSumReducer.class);
495
496 job.setNumReduceTasks(1);
497
498 job.setOutputFormatClass(TextOutputFormat.class);
499 TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
500
501 TableMapReduceUtil.addDependencyJars(job);
502 TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
503 Histogram.class,
504 Gson.class);
505
506 TableMapReduceUtil.initCredentials(job);
507
508 job.waitForCompletion(true);
509 return job;
510 }
511
512
513
514
515
516
517
518 private static Path writeInputFile(final Configuration c, final TestOptions opts) throws IOException {
519 SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
520 Path jobdir = new Path(PERF_EVAL_DIR, formatter.format(new Date()));
521 Path inputDir = new Path(jobdir, "inputs");
522
523 FileSystem fs = FileSystem.get(c);
524 fs.mkdirs(inputDir);
525
526 Path inputFile = new Path(inputDir, "input.txt");
527 PrintStream out = new PrintStream(fs.create(inputFile));
528
529 Map<Integer, String> m = new TreeMap<Integer, String>();
530 Hash h = MurmurHash.getInstance();
531 int perClientRows = (opts.totalRows / opts.numClientThreads);
532 try {
533 for (int i = 0; i < 10; i++) {
534 for (int j = 0; j < opts.numClientThreads; j++) {
535 TestOptions next = new TestOptions(opts);
536 next.startRow = (j * perClientRows) + (i * (perClientRows/10));
537 next.perClientRunRows = perClientRows / 10;
538 String s = GSON.toJson(next);
539 LOG.info("maptask input=" + s);
540 int hash = h.hash(Bytes.toBytes(s));
541 m.put(hash, s);
542 }
543 }
544 for (Map.Entry<Integer, String> e: m.entrySet()) {
545 out.println(e.getValue());
546 }
547 } finally {
548 out.close();
549 }
550 return inputDir;
551 }
552
553
554
555
556 static class CmdDescriptor {
557 private Class<? extends Test> cmdClass;
558 private String name;
559 private String description;
560
561 CmdDescriptor(Class<? extends Test> cmdClass, String name, String description) {
562 this.cmdClass = cmdClass;
563 this.name = name;
564 this.description = description;
565 }
566
567 public Class<? extends Test> getCmdClass() {
568 return cmdClass;
569 }
570
571 public String getName() {
572 return name;
573 }
574
575 public String getDescription() {
576 return description;
577 }
578 }
579
580
581
582
583
584
585
586
587 static class TestOptions {
588 String cmdName = null;
589 boolean nomapred = false;
590 boolean filterAll = false;
591 int startRow = 0;
592 float size = 1.0f;
593 int perClientRunRows = DEFAULT_ROWS_PER_GB;
594 int numClientThreads = 1;
595 int totalRows = DEFAULT_ROWS_PER_GB;
596 float sampleRate = 1.0f;
597 double traceRate = 0.0;
598 String tableName = TABLE_NAME;
599 boolean flushCommits = true;
600 boolean writeToWAL = true;
601 boolean autoFlush = false;
602 boolean oneCon = false;
603 boolean useTags = false;
604 int noOfTags = 1;
605 boolean reportLatency = false;
606 int multiGet = 0;
607 int randomSleep = 0;
608 boolean inMemoryCF = false;
609 int presplitRegions = 0;
610 int replicas = HTableDescriptor.DEFAULT_REGION_REPLICATION;
611 String splitPolicy = null;
612 Compression.Algorithm compression = Compression.Algorithm.NONE;
613 BloomType bloomType = BloomType.ROW;
614 int blockSize = HConstants.DEFAULT_BLOCKSIZE;
615 DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
616 boolean valueRandom = false;
617 boolean valueZipf = false;
618 int valueSize = DEFAULT_VALUE_LENGTH;
619 int period = (this.perClientRunRows / 10) == 0? perClientRunRows: perClientRunRows / 10;
620 int columns = 1;
621 int families = 1;
622 int caching = 30;
623 boolean addColumns = true;
624 long bufferSize = 2l * 1024l * 1024l;
625
626 public TestOptions() {}
627
628
629
630
631
632 public TestOptions(TestOptions that) {
633 this.cmdName = that.cmdName;
634 this.nomapred = that.nomapred;
635 this.startRow = that.startRow;
636 this.size = that.size;
637 this.perClientRunRows = that.perClientRunRows;
638 this.numClientThreads = that.numClientThreads;
639 this.totalRows = that.totalRows;
640 this.sampleRate = that.sampleRate;
641 this.traceRate = that.traceRate;
642 this.tableName = that.tableName;
643 this.flushCommits = that.flushCommits;
644 this.writeToWAL = that.writeToWAL;
645 this.autoFlush = that.autoFlush;
646 this.oneCon = that.oneCon;
647 this.useTags = that.useTags;
648 this.noOfTags = that.noOfTags;
649 this.reportLatency = that.reportLatency;
650 this.multiGet = that.multiGet;
651 this.inMemoryCF = that.inMemoryCF;
652 this.presplitRegions = that.presplitRegions;
653 this.replicas = that.replicas;
654 this.splitPolicy = that.splitPolicy;
655 this.compression = that.compression;
656 this.blockEncoding = that.blockEncoding;
657 this.filterAll = that.filterAll;
658 this.bloomType = that.bloomType;
659 this.blockSize = that.blockSize;
660 this.valueRandom = that.valueRandom;
661 this.valueZipf = that.valueZipf;
662 this.valueSize = that.valueSize;
663 this.period = that.period;
664 this.randomSleep = that.randomSleep;
665 this.addColumns = that.addColumns;
666 this.columns = that.columns;
667 this.families = that.families;
668 this.caching = that.caching;
669 this.bufferSize = that.bufferSize;
670 }
671
672 public int getCaching() {
673 return this.caching;
674 }
675
676 public void setCaching(final int caching) {
677 this.caching = caching;
678 }
679
680 public int getColumns() {
681 return this.columns;
682 }
683
684 public void setColumns(final int columns) {
685 this.columns = columns;
686 }
687
688 public int getFamilies() {
689 return this.families;
690 }
691
692 public void setFamilies(final int families) {
693 this.families = families;
694 }
695
696 public boolean isValueZipf() {
697 return valueZipf;
698 }
699
700 public void setValueZipf(boolean valueZipf) {
701 this.valueZipf = valueZipf;
702 }
703
704 public String getCmdName() {
705 return cmdName;
706 }
707
708 public void setCmdName(String cmdName) {
709 this.cmdName = cmdName;
710 }
711
712 public int getRandomSleep() {
713 return randomSleep;
714 }
715
716 public void setRandomSleep(int randomSleep) {
717 this.randomSleep = randomSleep;
718 }
719
720 public int getReplicas() {
721 return replicas;
722 }
723
724 public void setReplicas(int replicas) {
725 this.replicas = replicas;
726 }
727
728 public String getSplitPolicy() {
729 return splitPolicy;
730 }
731
732 public void setSplitPolicy(String splitPolicy) {
733 this.splitPolicy = splitPolicy;
734 }
735
736 public void setNomapred(boolean nomapred) {
737 this.nomapred = nomapred;
738 }
739
740 public void setFilterAll(boolean filterAll) {
741 this.filterAll = filterAll;
742 }
743
744 public void setStartRow(int startRow) {
745 this.startRow = startRow;
746 }
747
748 public void setSize(float size) {
749 this.size = size;
750 }
751
752 public void setPerClientRunRows(int perClientRunRows) {
753 this.perClientRunRows = perClientRunRows;
754 }
755
756 public void setNumClientThreads(int numClientThreads) {
757 this.numClientThreads = numClientThreads;
758 }
759
760 public void setTotalRows(int totalRows) {
761 this.totalRows = totalRows;
762 }
763
764 public void setSampleRate(float sampleRate) {
765 this.sampleRate = sampleRate;
766 }
767
768 public void setTraceRate(double traceRate) {
769 this.traceRate = traceRate;
770 }
771
772 public void setTableName(String tableName) {
773 this.tableName = tableName;
774 }
775
776 public void setFlushCommits(boolean flushCommits) {
777 this.flushCommits = flushCommits;
778 }
779
780 public void setWriteToWAL(boolean writeToWAL) {
781 this.writeToWAL = writeToWAL;
782 }
783
784 public void setAutoFlush(boolean autoFlush) {
785 this.autoFlush = autoFlush;
786 }
787
788 public void setOneCon(boolean oneCon) {
789 this.oneCon = oneCon;
790 }
791
792 public void setUseTags(boolean useTags) {
793 this.useTags = useTags;
794 }
795
796 public void setNoOfTags(int noOfTags) {
797 this.noOfTags = noOfTags;
798 }
799
800 public void setReportLatency(boolean reportLatency) {
801 this.reportLatency = reportLatency;
802 }
803
804 public void setMultiGet(int multiGet) {
805 this.multiGet = multiGet;
806 }
807
808 public void setInMemoryCF(boolean inMemoryCF) {
809 this.inMemoryCF = inMemoryCF;
810 }
811
812 public void setPresplitRegions(int presplitRegions) {
813 this.presplitRegions = presplitRegions;
814 }
815
816 public void setCompression(Compression.Algorithm compression) {
817 this.compression = compression;
818 }
819
820 public void setBloomType(BloomType bloomType) {
821 this.bloomType = bloomType;
822 }
823
824 public void setBlockSize(int blockSize) {
825 this.blockSize = blockSize;
826 }
827
828 public void setBlockEncoding(DataBlockEncoding blockEncoding) {
829 this.blockEncoding = blockEncoding;
830 }
831
832 public void setValueRandom(boolean valueRandom) {
833 this.valueRandom = valueRandom;
834 }
835
836 public void setValueSize(int valueSize) {
837 this.valueSize = valueSize;
838 }
839
840 public void setBufferSize(long bufferSize) {
841 this.bufferSize = bufferSize;
842 }
843
844 public long getBufferSize() {
845 return this.bufferSize;
846 }
847
848 public void setPeriod(int period) {
849 this.period = period;
850 }
851
852 public boolean isNomapred() {
853 return nomapred;
854 }
855
856 public boolean isFilterAll() {
857 return filterAll;
858 }
859
860 public int getStartRow() {
861 return startRow;
862 }
863
864 public float getSize() {
865 return size;
866 }
867
868 public int getPerClientRunRows() {
869 return perClientRunRows;
870 }
871
872 public int getNumClientThreads() {
873 return numClientThreads;
874 }
875
876 public int getTotalRows() {
877 return totalRows;
878 }
879
880 public float getSampleRate() {
881 return sampleRate;
882 }
883
884 public double getTraceRate() {
885 return traceRate;
886 }
887
888 public String getTableName() {
889 return tableName;
890 }
891
892 public boolean isFlushCommits() {
893 return flushCommits;
894 }
895
896 public boolean isWriteToWAL() {
897 return writeToWAL;
898 }
899
900 public boolean isAutoFlush() {
901 return autoFlush;
902 }
903
904 public boolean isUseTags() {
905 return useTags;
906 }
907
908 public int getNoOfTags() {
909 return noOfTags;
910 }
911
912 public boolean isReportLatency() {
913 return reportLatency;
914 }
915
916 public int getMultiGet() {
917 return multiGet;
918 }
919
920 public boolean isInMemoryCF() {
921 return inMemoryCF;
922 }
923
924 public int getPresplitRegions() {
925 return presplitRegions;
926 }
927
928 public Compression.Algorithm getCompression() {
929 return compression;
930 }
931
932 public DataBlockEncoding getBlockEncoding() {
933 return blockEncoding;
934 }
935
936 public boolean isValueRandom() {
937 return valueRandom;
938 }
939
940 public int getValueSize() {
941 return valueSize;
942 }
943
944 public int getPeriod() {
945 return period;
946 }
947
948 public BloomType getBloomType() {
949 return bloomType;
950 }
951
952 public int getBlockSize() {
953 return blockSize;
954 }
955
956 public boolean isOneCon() {
957 return oneCon;
958 }
959
960 public boolean getAddColumns() {
961 return addColumns;
962 }
963
964 public void setAddColumns(boolean addColumns) {
965 this.addColumns = addColumns;
966 }
967 }
968
969
970
971
972
973 static abstract class Test {
974
975
976 private static final Random randomSeed = new Random(System.currentTimeMillis());
977
978 private static long nextRandomSeed() {
979 return randomSeed.nextLong();
980 }
981 private final int everyN;
982
983 protected final Random rand = new Random(nextRandomSeed());
984 protected final Configuration conf;
985 protected final TestOptions opts;
986
987 private final Status status;
988 private final Sampler<?> traceSampler;
989 private final SpanReceiverHost receiverHost;
990 protected Connection connection;
991
992 private String testName;
993 private Histogram latencyHistogram;
994 private Histogram valueSizeHistogram;
995 private Histogram rpcCallsHistogram;
996 private Histogram remoteRpcCallsHistogram;
997 private Histogram millisBetweenNextHistogram;
998 private Histogram regionsScannedHistogram;
999 private Histogram bytesInResultsHistogram;
1000 private Histogram bytesInRemoteResultsHistogram;
1001 private RandomDistribution.Zipf zipf;
1002
1003
1004
1005
1006
1007 Test(final Connection con, final TestOptions options, final Status status) {
1008 this.connection = con;
1009 this.conf = con == null ? HBaseConfiguration.create() : this.connection.getConfiguration();
1010 this.opts = options;
1011 this.status = status;
1012 this.testName = this.getClass().getSimpleName();
1013 receiverHost = SpanReceiverHost.getInstance(conf);
1014 if (options.traceRate >= 1.0) {
1015 this.traceSampler = Sampler.ALWAYS;
1016 } else if (options.traceRate > 0.0) {
1017 conf.setDouble("hbase.sampler.fraction", options.traceRate);
1018 this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
1019 } else {
1020 this.traceSampler = Sampler.NEVER;
1021 }
1022 everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
1023 if (options.isValueZipf()) {
1024 this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.1);
1025 }
1026 LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
1027 }
1028
1029 int getValueLength(final Random r) {
1030 if (this.opts.isValueRandom()) {
1031 return r.nextInt(opts.valueSize);
1032 } else if (this.opts.isValueZipf()) {
1033 return Math.abs(this.zipf.nextInt());
1034 } else {
1035 return opts.valueSize;
1036 }
1037 }
1038
1039 void updateValueSize(final Result [] rs) throws IOException {
1040 if (rs == null || !isRandomValueSize()) return;
1041 for (Result r: rs) updateValueSize(r);
1042 }
1043
1044 void updateValueSize(final Result r) throws IOException {
1045 if (r == null || !isRandomValueSize()) return;
1046 int size = 0;
1047 for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
1048 size += scanner.current().getValueLength();
1049 }
1050 updateValueSize(size);
1051 }
1052
1053 void updateValueSize(final int valueSize) {
1054 if (!isRandomValueSize()) return;
1055 this.valueSizeHistogram.update(valueSize);
1056 }
1057
1058 void updateScanMetrics(final ScanMetrics metrics) {
1059 if (metrics == null) return;
1060 Map<String,Long> metricsMap = metrics.getMetricsMap();
1061 Long rpcCalls = metricsMap.get(ScanMetrics.RPC_CALLS_METRIC_NAME);
1062 if (rpcCalls != null) {
1063 this.rpcCallsHistogram.update(rpcCalls.longValue());
1064 }
1065 Long remoteRpcCalls = metricsMap.get(ScanMetrics.REMOTE_RPC_CALLS_METRIC_NAME);
1066 if (remoteRpcCalls != null) {
1067 this.remoteRpcCallsHistogram.update(remoteRpcCalls.longValue());
1068 }
1069 Long millisBetweenNext = metricsMap.get(ScanMetrics.MILLIS_BETWEEN_NEXTS_METRIC_NAME);
1070 if (millisBetweenNext != null) {
1071 this.millisBetweenNextHistogram.update(millisBetweenNext.longValue());
1072 }
1073 Long regionsScanned = metricsMap.get(ScanMetrics.REGIONS_SCANNED_METRIC_NAME);
1074 if (regionsScanned != null) {
1075 this.regionsScannedHistogram.update(regionsScanned.longValue());
1076 }
1077 Long bytesInResults = metricsMap.get(ScanMetrics.BYTES_IN_RESULTS_METRIC_NAME);
1078 if (bytesInResults != null && bytesInResults.longValue() > 0) {
1079 this.bytesInResultsHistogram.update(bytesInResults.longValue());
1080 }
1081 Long bytesInRemoteResults = metricsMap.get(ScanMetrics.BYTES_IN_REMOTE_RESULTS_METRIC_NAME);
1082 if (bytesInRemoteResults != null && bytesInRemoteResults.longValue() > 0) {
1083 this.bytesInRemoteResultsHistogram.update(bytesInRemoteResults.longValue());
1084 }
1085 }
1086
1087 String generateStatus(final int sr, final int i, final int lr) {
1088 return sr + "/" + i + "/" + lr + ", latency " + getShortLatencyReport() +
1089 (!isRandomValueSize()? "": ", value size " + getShortValueSizeReport());
1090 }
1091
1092 boolean isRandomValueSize() {
1093 return opts.valueRandom;
1094 }
1095
1096 protected int getReportingPeriod() {
1097 return opts.period;
1098 }
1099
1100
1101
1102
1103 public Histogram getLatencyHistogram() {
1104 return latencyHistogram;
1105 }
1106
1107 void testSetup() throws IOException {
1108
1109 latencyHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1110 valueSizeHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1111
1112 rpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1113 remoteRpcCallsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1114 millisBetweenNextHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1115 regionsScannedHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1116 bytesInResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1117 bytesInRemoteResultsHistogram = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500));
1118
1119 if (!opts.oneCon) {
1120 this.connection = ConnectionFactory.createConnection(conf);
1121 }
1122
1123 onStartup();
1124 }
1125
1126 abstract void onStartup() throws IOException;
1127
1128 void testTakedown() throws IOException {
1129 onTakedown();
1130
1131
1132
1133 synchronized (Test.class) {
1134 status.setStatus("Test : " + testName + ", Thread : " + Thread.currentThread().getName());
1135 status.setStatus("Latency (us) : " + YammerHistogramUtils.getHistogramReport(
1136 latencyHistogram));
1137 status.setStatus("Num measures (latency) : " + latencyHistogram.count());
1138 status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(latencyHistogram));
1139 status.setStatus("ValueSize (bytes) : "
1140 + YammerHistogramUtils.getHistogramReport(valueSizeHistogram));
1141 status.setStatus("Num measures (ValueSize): " + valueSizeHistogram.count());
1142 status.setStatus(YammerHistogramUtils.getPrettyHistogramReport(valueSizeHistogram));
1143 if (rpcCallsHistogram.count() > 0) {
1144 status.setStatus("rpcCalls (count): " +
1145 YammerHistogramUtils.getHistogramReport(rpcCallsHistogram));
1146 }
1147 if (remoteRpcCallsHistogram.count() > 0) {
1148 status.setStatus("remoteRpcCalls (count): " +
1149 YammerHistogramUtils.getHistogramReport(remoteRpcCallsHistogram));
1150 }
1151 if (millisBetweenNextHistogram.count() > 0) {
1152 status.setStatus("millisBetweenNext (latency): " +
1153 YammerHistogramUtils.getHistogramReport(millisBetweenNextHistogram));
1154 }
1155 if (regionsScannedHistogram.count() > 0) {
1156 status.setStatus("regionsScanned (count): " +
1157 YammerHistogramUtils.getHistogramReport(regionsScannedHistogram));
1158 }
1159 if (bytesInResultsHistogram.count() > 0) {
1160 status.setStatus("bytesInResults (size): " +
1161 YammerHistogramUtils.getHistogramReport(bytesInResultsHistogram));
1162 }
1163 if (bytesInRemoteResultsHistogram.count() > 0) {
1164 status.setStatus("bytesInRemoteResults (size): " +
1165 YammerHistogramUtils.getHistogramReport(bytesInRemoteResultsHistogram));
1166 }
1167 }
1168 if (!opts.oneCon) {
1169 connection.close();
1170 }
1171 receiverHost.closeReceivers();
1172 }
1173
1174 abstract void onTakedown() throws IOException;
1175
1176
1177
1178
1179
1180
1181 long test() throws IOException, InterruptedException {
1182 testSetup();
1183 LOG.info("Timed test starting in thread " + Thread.currentThread().getName());
1184 final long startTime = System.nanoTime();
1185 try {
1186 testTimed();
1187 } finally {
1188 testTakedown();
1189 }
1190 return (System.nanoTime() - startTime) / 1000000;
1191 }
1192
1193 int getStartRow() {
1194 return opts.startRow;
1195 }
1196
1197 int getLastRow() {
1198 return getStartRow() + opts.perClientRunRows;
1199 }
1200
1201
1202
1203
1204 void testTimed() throws IOException, InterruptedException {
1205 int startRow = getStartRow();
1206 int lastRow = getLastRow();
1207
1208 for (int i = startRow; i < lastRow; i++) {
1209 if (i % everyN != 0) continue;
1210 long startTime = System.nanoTime();
1211 TraceScope scope = Trace.startSpan("test row", traceSampler);
1212 try {
1213 testRow(i);
1214 } finally {
1215 scope.close();
1216 }
1217
1218
1219
1220 if (opts.multiGet == 0 || (i - startRow + 1) % opts.multiGet == 0) {
1221 latencyHistogram.update((System.nanoTime() - startTime) / 1000);
1222 }
1223 if (status != null && i > 0 && (i % getReportingPeriod()) == 0) {
1224 status.setStatus(generateStatus(startRow, i, lastRow));
1225 }
1226 }
1227 }
1228
1229
1230
1231
1232 public String getShortLatencyReport() {
1233 return YammerHistogramUtils.getShortHistogramReport(this.latencyHistogram);
1234 }
1235
1236
1237
1238
1239 public String getShortValueSizeReport() {
1240 return YammerHistogramUtils.getShortHistogramReport(this.valueSizeHistogram);
1241 }
1242
1243
1244
1245
1246
1247 abstract void testRow(final int i) throws IOException, InterruptedException;
1248 }
1249
1250 static abstract class TableTest extends Test {
1251 protected Table table;
1252
1253 TableTest(Connection con, TestOptions options, Status status) {
1254 super(con, options, status);
1255 }
1256
1257 @Override
1258 void onStartup() throws IOException {
1259 this.table = connection.getTable(TableName.valueOf(opts.tableName));
1260 }
1261
1262 @Override
1263 void onTakedown() throws IOException {
1264 table.close();
1265 }
1266 }
1267
1268 static abstract class BufferedMutatorTest extends Test {
1269 protected BufferedMutator mutator;
1270
1271 BufferedMutatorTest(Connection con, TestOptions options, Status status) {
1272 super(con, options, status);
1273 }
1274
1275 @Override
1276 void onStartup() throws IOException {
1277 BufferedMutatorParams p = new BufferedMutatorParams(TableName.valueOf(opts.tableName));
1278 p.writeBufferSize(opts.bufferSize);
1279 this.mutator = connection.getBufferedMutator(p);
1280 }
1281
1282 @Override
1283 void onTakedown() throws IOException {
1284 mutator.close();
1285 }
1286 }
1287
1288 static class RandomSeekScanTest extends TableTest {
1289 RandomSeekScanTest(Connection con, TestOptions options, Status status) {
1290 super(con, options, status);
1291 }
1292
1293 @Override
1294 void testRow(final int i) throws IOException {
1295 Scan scan = new Scan(getRandomRow(this.rand, opts.totalRows));
1296 FilterList list = new FilterList();
1297 for (int family = 0; family < opts.families; family++) {
1298 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1299 if (opts.addColumns) {
1300 for (int column = 0; column < opts.columns; column++) {
1301 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1302 scan.addColumn(familyName, qualifier);
1303 }
1304 } else {
1305 scan.addFamily(familyName);
1306 }
1307 }
1308 if (opts.filterAll) {
1309 list.addFilter(new FilterAllFilter());
1310 }
1311 list.addFilter(new WhileMatchFilter(new PageFilter(120)));
1312 scan.setFilter(list);
1313 scan.setCaching(opts.caching);
1314 scan.setScanMetricsEnabled(true);
1315 ResultScanner s = this.table.getScanner(scan);
1316 try {
1317 for (Result rr; (rr = s.next()) != null;) {
1318 updateValueSize(rr);
1319 }
1320 } finally {
1321 if (s != null) {
1322 updateScanMetrics(scan.getScanMetrics());
1323 s.close();
1324 }
1325 }
1326 }
1327
1328 @Override
1329 protected int getReportingPeriod() {
1330 int period = opts.perClientRunRows / 100;
1331 return period == 0 ? opts.perClientRunRows : period;
1332 }
1333
1334 }
1335
1336 static abstract class RandomScanWithRangeTest extends TableTest {
1337 RandomScanWithRangeTest(Connection con, TestOptions options, Status status) {
1338 super(con, options, status);
1339 }
1340
1341 @Override
1342 void testRow(final int i) throws IOException {
1343 Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
1344 Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
1345 for (int family = 0; family < opts.families; family++) {
1346 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1347 if (opts.addColumns) {
1348 for (int column = 0; column < opts.columns; column++) {
1349 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1350 scan.addColumn(familyName, qualifier);
1351 }
1352 } else {
1353 scan.addFamily(familyName);
1354 }
1355 }
1356 if (opts.filterAll) {
1357 scan.setFilter(new FilterAllFilter());
1358 }
1359 scan.setCaching(opts.caching);
1360 scan.setScanMetricsEnabled(true);
1361 Result r = null;
1362 int count = 0;
1363 ResultScanner s = this.table.getScanner(scan);
1364 try {
1365 for (; (r = s.next()) != null;) {
1366 updateValueSize(r);
1367 count++;
1368 }
1369 if (i % 100 == 0) {
1370 LOG.info(String.format("Scan for key range %s - %s returned %s rows",
1371 Bytes.toString(startAndStopRow.getFirst()),
1372 Bytes.toString(startAndStopRow.getSecond()), count));
1373 }
1374 } finally {
1375 if (s != null) {
1376 updateScanMetrics(scan.getScanMetrics());
1377 s.close();
1378 }
1379 }
1380 }
1381
1382 protected abstract Pair<byte[],byte[]> getStartAndStopRow();
1383
1384 protected Pair<byte[], byte[]> generateStartAndStopRows(int maxRange) {
1385 int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows;
1386 int stop = start + maxRange;
1387 return new Pair<byte[],byte[]>(format(start), format(stop));
1388 }
1389
1390 @Override
1391 protected int getReportingPeriod() {
1392 int period = opts.perClientRunRows / 100;
1393 return period == 0? opts.perClientRunRows: period;
1394 }
1395 }
1396
1397 static class RandomScanWithRange10Test extends RandomScanWithRangeTest {
1398 RandomScanWithRange10Test(Connection con, TestOptions options, Status status) {
1399 super(con, options, status);
1400 }
1401
1402 @Override
1403 protected Pair<byte[], byte[]> getStartAndStopRow() {
1404 return generateStartAndStopRows(10);
1405 }
1406 }
1407
1408 static class RandomScanWithRange100Test extends RandomScanWithRangeTest {
1409 RandomScanWithRange100Test(Connection con, TestOptions options, Status status) {
1410 super(con, options, status);
1411 }
1412
1413 @Override
1414 protected Pair<byte[], byte[]> getStartAndStopRow() {
1415 return generateStartAndStopRows(100);
1416 }
1417 }
1418
1419 static class RandomScanWithRange1000Test extends RandomScanWithRangeTest {
1420 RandomScanWithRange1000Test(Connection con, TestOptions options, Status status) {
1421 super(con, options, status);
1422 }
1423
1424 @Override
1425 protected Pair<byte[], byte[]> getStartAndStopRow() {
1426 return generateStartAndStopRows(1000);
1427 }
1428 }
1429
1430 static class RandomScanWithRange10000Test extends RandomScanWithRangeTest {
1431 RandomScanWithRange10000Test(Connection con, TestOptions options, Status status) {
1432 super(con, options, status);
1433 }
1434
1435 @Override
1436 protected Pair<byte[], byte[]> getStartAndStopRow() {
1437 return generateStartAndStopRows(10000);
1438 }
1439 }
1440
1441 static class RandomReadTest extends TableTest {
1442 private final Consistency consistency;
1443 private ArrayList<Get> gets;
1444 private Random rd = new Random();
1445
1446 RandomReadTest(Connection con, TestOptions options, Status status) {
1447 super(con, options, status);
1448 consistency = options.replicas == DEFAULT_OPTS.replicas ? null : Consistency.TIMELINE;
1449 if (opts.multiGet > 0) {
1450 LOG.info("MultiGet enabled. Sending GETs in batches of " + opts.multiGet + ".");
1451 this.gets = new ArrayList<Get>(opts.multiGet);
1452 }
1453 }
1454
1455 @Override
1456 void testRow(final int i) throws IOException, InterruptedException {
1457 if (opts.randomSleep > 0) {
1458 Thread.sleep(rd.nextInt(opts.randomSleep));
1459 }
1460 Get get = new Get(getRandomRow(this.rand, opts.totalRows));
1461 for (int family = 0; family < opts.families; family++) {
1462 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1463 if (opts.addColumns) {
1464 for (int column = 0; column < opts.columns; column++) {
1465 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1466 get.addColumn(familyName, qualifier);
1467 }
1468 } else {
1469 get.addFamily(familyName);
1470 }
1471 }
1472 if (opts.filterAll) {
1473 get.setFilter(new FilterAllFilter());
1474 }
1475 get.setConsistency(consistency);
1476 if (LOG.isTraceEnabled()) LOG.trace(get.toString());
1477 if (opts.multiGet > 0) {
1478 this.gets.add(get);
1479 if (this.gets.size() == opts.multiGet) {
1480 Result [] rs = this.table.get(this.gets);
1481 updateValueSize(rs);
1482 this.gets.clear();
1483 }
1484 } else {
1485 updateValueSize(this.table.get(get));
1486 }
1487 }
1488
1489 @Override
1490 protected int getReportingPeriod() {
1491 int period = opts.perClientRunRows / 10;
1492 return period == 0 ? opts.perClientRunRows : period;
1493 }
1494
1495 @Override
1496 protected void testTakedown() throws IOException {
1497 if (this.gets != null && this.gets.size() > 0) {
1498 this.table.get(gets);
1499 this.gets.clear();
1500 }
1501 super.testTakedown();
1502 }
1503 }
1504
1505 static class RandomWriteTest extends BufferedMutatorTest {
1506 RandomWriteTest(Connection con, TestOptions options, Status status) {
1507 super(con, options, status);
1508 }
1509
1510 @Override
1511 void testRow(final int i) throws IOException {
1512 byte[] row = getRandomRow(this.rand, opts.totalRows);
1513 Put put = new Put(row);
1514 for (int family = 0; family < opts.families; family++) {
1515 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1516 for (int column = 0; column < opts.columns; column++) {
1517 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1518 byte[] value = generateData(this.rand, getValueLength(this.rand));
1519 if (opts.useTags) {
1520 byte[] tag = generateData(this.rand, TAG_LENGTH);
1521 Tag[] tags = new Tag[opts.noOfTags];
1522 for (int n = 0; n < opts.noOfTags; n++) {
1523 Tag t = new Tag((byte) n, tag);
1524 tags[n] = t;
1525 }
1526 KeyValue kv = new KeyValue(row, familyName, qualifier, HConstants.LATEST_TIMESTAMP,
1527 value, tags);
1528 put.add(kv);
1529 updateValueSize(kv.getValueLength());
1530 } else {
1531 put.add(familyName, qualifier, value);
1532 updateValueSize(value.length);
1533 }
1534 }
1535 }
1536 put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1537 mutator.mutate(put);
1538 if (opts.autoFlush) {
1539 mutator.flush();
1540 }
1541 }
1542 }
1543
1544 static class ScanTest extends TableTest {
1545 private Scan scan;
1546 private ResultScanner testScanner;
1547
1548 ScanTest(Connection con, TestOptions options, Status status) {
1549 super(con, options, status);
1550 }
1551
1552 @Override
1553 void testTakedown() throws IOException {
1554 if (this.testScanner != null) {
1555 this.testScanner.close();
1556 }
1557 super.testTakedown();
1558 }
1559
1560
1561 @Override
1562 void testRow(final int i) throws IOException {
1563 if (this.testScanner == null) {
1564 scan = new Scan(format(opts.startRow));
1565 for (int family = 0; family < opts.families; family++) {
1566 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1567 if (opts.addColumns) {
1568 for (int column = 0; column < opts.columns; column++) {
1569 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1570 scan.addColumn(familyName, qualifier);
1571 }
1572 } else {
1573 scan.addFamily(familyName);
1574 }
1575 }
1576 if (opts.filterAll) {
1577 scan.setFilter(new FilterAllFilter());
1578 }
1579 scan.setCaching(opts.caching);
1580 scan.setScanMetricsEnabled(true);
1581 this.testScanner = table.getScanner(scan);
1582 }
1583 try {
1584 Result r = testScanner.next();
1585 updateValueSize(r);
1586 } finally {
1587 updateScanMetrics(scan.getScanMetrics());
1588 }
1589 }
1590 }
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600 static abstract class CASTableTest extends TableTest {
1601 private final byte [] qualifier;
1602 CASTableTest(Connection con, TestOptions options, Status status) {
1603 super(con, options, status);
1604 qualifier = Bytes.toBytes(this.getClass().getSimpleName());
1605 }
1606
1607 byte [] getQualifier() {
1608 return this.qualifier;
1609 }
1610
1611 @Override
1612 int getStartRow() {
1613 return 0;
1614 }
1615
1616 @Override
1617 int getLastRow() {
1618 return opts.perClientRunRows;
1619 }
1620 }
1621
1622 static class IncrementTest extends CASTableTest {
1623 IncrementTest(Connection con, TestOptions options, Status status) {
1624 super(con, options, status);
1625 }
1626
1627 @Override
1628 void testRow(final int i) throws IOException {
1629 Increment increment = new Increment(format(i));
1630
1631
1632
1633 for (int family = 0; family < opts.families; family++) {
1634 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1635 increment.addColumn(familyName, getQualifier(), 1l);
1636 }
1637 updateValueSize(this.table.increment(increment));
1638 }
1639 }
1640
1641 static class AppendTest extends CASTableTest {
1642 AppendTest(Connection con, TestOptions options, Status status) {
1643 super(con, options, status);
1644 }
1645
1646 @Override
1647 void testRow(final int i) throws IOException {
1648 byte [] bytes = format(i);
1649 Append append = new Append(bytes);
1650
1651
1652
1653 for (int family = 0; family < opts.families; family++) {
1654 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1655 append.add(familyName, getQualifier(), bytes);
1656 }
1657 updateValueSize(this.table.append(append));
1658 }
1659 }
1660
1661 static class CheckAndMutateTest extends CASTableTest {
1662 CheckAndMutateTest(Connection con, TestOptions options, Status status) {
1663 super(con, options, status);
1664 }
1665
1666 @Override
1667 void testRow(final int i) throws IOException {
1668 final byte [] bytes = format(i);
1669
1670
1671 Put put = new Put(bytes);
1672 put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1673 this.table.put(put);
1674 RowMutations mutations = new RowMutations(bytes);
1675 mutations.add(put);
1676 this.table.checkAndMutate(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes,
1677 mutations);
1678 }
1679 }
1680
1681 static class CheckAndPutTest extends CASTableTest {
1682 CheckAndPutTest(Connection con, TestOptions options, Status status) {
1683 super(con, options, status);
1684 }
1685
1686 @Override
1687 void testRow(final int i) throws IOException {
1688 final byte [] bytes = format(i);
1689
1690
1691 Put put = new Put(bytes);
1692 put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1693 this.table.put(put);
1694 this.table.checkAndPut(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes, put);
1695 }
1696 }
1697
1698 static class CheckAndDeleteTest extends CASTableTest {
1699 CheckAndDeleteTest(Connection con, TestOptions options, Status status) {
1700 super(con, options, status);
1701 }
1702
1703 @Override
1704 void testRow(final int i) throws IOException {
1705 final byte [] bytes = format(i);
1706
1707
1708 Put put = new Put(bytes);
1709 put.addColumn(FAMILY_ZERO, getQualifier(), bytes);
1710 this.table.put(put);
1711 Delete delete = new Delete(put.getRow());
1712 delete.addColumn(FAMILY_ZERO, getQualifier());
1713 this.table.checkAndDelete(bytes, FAMILY_ZERO, getQualifier(), CompareOp.EQUAL, bytes, delete);
1714 }
1715 }
1716
1717 static class SequentialReadTest extends TableTest {
1718 SequentialReadTest(Connection con, TestOptions options, Status status) {
1719 super(con, options, status);
1720 }
1721
1722 @Override
1723 void testRow(final int i) throws IOException {
1724 Get get = new Get(format(i));
1725 for (int family = 0; family < opts.families; family++) {
1726 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1727 if (opts.addColumns) {
1728 for (int column = 0; column < opts.columns; column++) {
1729 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1730 get.addColumn(familyName, qualifier);
1731 }
1732 } else {
1733 get.addFamily(familyName);
1734 }
1735 }
1736 if (opts.filterAll) {
1737 get.setFilter(new FilterAllFilter());
1738 }
1739 updateValueSize(table.get(get));
1740 }
1741 }
1742
1743 static class SequentialWriteTest extends BufferedMutatorTest {
1744 SequentialWriteTest(Connection con, TestOptions options, Status status) {
1745 super(con, options, status);
1746 }
1747
1748 @Override
1749 void testRow(final int i) throws IOException {
1750 byte[] row = format(i);
1751 Put put = new Put(row);
1752 for (int family = 0; family < opts.families; family++) {
1753 byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
1754 for (int column = 0; column < opts.columns; column++) {
1755 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1756 byte[] value = generateData(this.rand, getValueLength(this.rand));
1757 if (opts.useTags) {
1758 byte[] tag = generateData(this.rand, TAG_LENGTH);
1759 Tag[] tags = new Tag[opts.noOfTags];
1760 for (int n = 0; n < opts.noOfTags; n++) {
1761 Tag t = new Tag((byte) n, tag);
1762 tags[n] = t;
1763 }
1764 KeyValue kv = new KeyValue(row, familyName, qualifier, HConstants.LATEST_TIMESTAMP,
1765 value, tags);
1766 put.add(kv);
1767 updateValueSize(kv.getValueLength());
1768 } else {
1769 put.add(familyName, qualifier, value);
1770 updateValueSize(value.length);
1771 }
1772 }
1773 }
1774 put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
1775 mutator.mutate(put);
1776 if (opts.autoFlush) {
1777 mutator.flush();
1778 }
1779 }
1780 }
1781
1782 static class FilteredScanTest extends TableTest {
1783 protected static final Log LOG = LogFactory.getLog(FilteredScanTest.class.getName());
1784
1785 FilteredScanTest(Connection con, TestOptions options, Status status) {
1786 super(con, options, status);
1787 if (opts.perClientRunRows == DEFAULT_ROWS_PER_GB) {
1788 LOG.warn("Option \"rows\" unspecified. Using default value " + DEFAULT_ROWS_PER_GB
1789 + ". This could take a very long time.");
1790 }
1791 }
1792
1793 @Override
1794 void testRow(int i) throws IOException {
1795 byte[] value = generateData(this.rand, getValueLength(this.rand));
1796 Scan scan = constructScan(value);
1797 ResultScanner scanner = null;
1798 try {
1799 scanner = this.table.getScanner(scan);
1800 for (Result r = null; (r = scanner.next()) != null;) {
1801 updateValueSize(r);
1802 }
1803 } finally {
1804 if (scanner != null) {
1805 updateScanMetrics(scan.getScanMetrics());
1806 scanner.close();
1807 }
1808 }
1809 }
1810
1811 protected Scan constructScan(byte[] valuePrefix) throws IOException {
1812 FilterList list = new FilterList();
1813 Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO,
1814 CompareFilter.CompareOp.EQUAL, new BinaryComparator(valuePrefix));
1815 list.addFilter(filter);
1816 if (opts.filterAll) {
1817 list.addFilter(new FilterAllFilter());
1818 }
1819 Scan scan = new Scan();
1820 if (opts.addColumns) {
1821 for (int column = 0; column < opts.columns; column++) {
1822 byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
1823 scan.addColumn(FAMILY_ZERO, qualifier);
1824 }
1825 } else {
1826 scan.addFamily(FAMILY_ZERO);
1827 }
1828 scan.setFilter(list);
1829 scan.setCaching(opts.caching);
1830 scan.setScanMetricsEnabled(true);
1831 return scan;
1832 }
1833 }
1834
1835
1836
1837
1838
1839
1840
1841 private static String calculateMbps(int rows, long timeMs, final int valueSize, int families,
1842 int columns) {
1843 BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH +
1844 ((valueSize + (FAMILY_NAME_BASE.length()+1) + COLUMN_ZERO.length) * columns) * families);
1845 BigDecimal mbps = BigDecimal.valueOf(rows).multiply(rowSize, CXT)
1846 .divide(BigDecimal.valueOf(timeMs), CXT).multiply(MS_PER_SEC, CXT)
1847 .divide(BYTES_PER_MB, CXT);
1848 return FMT.format(mbps) + " MB/s";
1849 }
1850
1851
1852
1853
1854
1855
1856
1857 public static byte [] format(final int number) {
1858 byte [] b = new byte[ROW_LENGTH];
1859 int d = Math.abs(number);
1860 for (int i = b.length - 1; i >= 0; i--) {
1861 b[i] = (byte)((d % 10) + '0');
1862 d /= 10;
1863 }
1864 return b;
1865 }
1866
1867
1868
1869
1870
1871
1872
1873 public static byte[] generateData(final Random r, int length) {
1874 byte [] b = new byte [length];
1875 int i;
1876
1877 for(i = 0; i < (length-8); i += 8) {
1878 b[i] = (byte) (65 + r.nextInt(26));
1879 b[i+1] = b[i];
1880 b[i+2] = b[i];
1881 b[i+3] = b[i];
1882 b[i+4] = b[i];
1883 b[i+5] = b[i];
1884 b[i+6] = b[i];
1885 b[i+7] = b[i];
1886 }
1887
1888 byte a = (byte) (65 + r.nextInt(26));
1889 for(; i < length; i++) {
1890 b[i] = a;
1891 }
1892 return b;
1893 }
1894
1895
1896
1897
1898
1899 @Deprecated
1900 public static byte[] generateValue(final Random r) {
1901 return generateData(r, DEFAULT_VALUE_LENGTH);
1902 }
1903
1904 static byte [] getRandomRow(final Random random, final int totalRows) {
1905 return format(random.nextInt(Integer.MAX_VALUE) % totalRows);
1906 }
1907
1908 static RunResult runOneClient(final Class<? extends Test> cmd, Configuration conf, Connection con,
1909 TestOptions opts, final Status status)
1910 throws IOException, InterruptedException {
1911 status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " +
1912 opts.perClientRunRows + " rows");
1913 long totalElapsedTime;
1914
1915 final Test t;
1916 try {
1917 Constructor<? extends Test> constructor =
1918 cmd.getDeclaredConstructor(Connection.class, TestOptions.class, Status.class);
1919 t = constructor.newInstance(con, opts, status);
1920 } catch (NoSuchMethodException e) {
1921 throw new IllegalArgumentException("Invalid command class: " +
1922 cmd.getName() + ". It does not provide a constructor as described by " +
1923 "the javadoc comment. Available constructors are: " +
1924 Arrays.toString(cmd.getConstructors()));
1925 } catch (Exception e) {
1926 throw new IllegalStateException("Failed to construct command class", e);
1927 }
1928 totalElapsedTime = t.test();
1929
1930 status.setStatus("Finished " + cmd + " in " + totalElapsedTime +
1931 "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" +
1932 " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime,
1933 getAverageValueLength(opts), opts.families, opts.columns) + ")");
1934
1935 return new RunResult(totalElapsedTime, t.getLatencyHistogram());
1936 }
1937
1938 private static int getAverageValueLength(final TestOptions opts) {
1939 return opts.valueRandom? opts.valueSize/2: opts.valueSize;
1940 }
1941
1942 private void runTest(final Class<? extends Test> cmd, TestOptions opts) throws IOException,
1943 InterruptedException, ClassNotFoundException {
1944
1945
1946 LOG.info(cmd.getSimpleName() + " test run options=" + GSON.toJson(opts));
1947 try(Connection conn = ConnectionFactory.createConnection(getConf());
1948 Admin admin = conn.getAdmin()) {
1949 checkTable(admin, opts);
1950 }
1951 if (opts.nomapred) {
1952 doLocalClients(opts, getConf());
1953 } else {
1954 doMapReduce(opts, getConf());
1955 }
1956 }
1957
1958 protected void printUsage() {
1959 printUsage(this.getClass().getName(), null);
1960 }
1961
1962 protected static void printUsage(final String message) {
1963 printUsage(PerformanceEvaluation.class.getName(), message);
1964 }
1965
1966 protected static void printUsageAndExit(final String message, final int exitCode) {
1967 printUsage(message);
1968 System.exit(exitCode);
1969 }
1970
1971 protected static void printUsage(final String className, final String message) {
1972 if (message != null && message.length() > 0) {
1973 System.err.println(message);
1974 }
1975 System.err.println("Usage: java " + className + " \\");
1976 System.err.println(" <OPTIONS> [-D<property=value>]* <command> <nclients>");
1977 System.err.println();
1978 System.err.println("Options:");
1979 System.err.println(" nomapred Run multiple clients using threads " +
1980 "(rather than use mapreduce)");
1981 System.err.println(" rows Rows each client runs. Default: " +
1982 DEFAULT_OPTS.getPerClientRunRows());
1983 System.err.println(" size Total size in GiB. Mutually exclusive with --rows. " +
1984 "Default: 1.0.");
1985 System.err.println(" sampleRate Execute test on a sample of total " +
1986 "rows. Only supported by randomRead. Default: 1.0");
1987 System.err.println(" traceRate Enable HTrace spans. Initiate tracing every N rows. " +
1988 "Default: 0");
1989 System.err.println(" table Alternate table name. Default: 'TestTable'");
1990 System.err.println(" multiGet If >0, when doing RandomRead, perform multiple gets " +
1991 "instead of single gets. Default: 0");
1992 System.err.println(" compress Compression type to use (GZ, LZO, ...). Default: 'NONE'");
1993 System.err.println(" flushCommits Used to determine if the test should flush the table. " +
1994 "Default: false");
1995 System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
1996 System.err.println(" autoFlush Set autoFlush on htable. Default: False");
1997 System.err.println(" oneCon all the threads share the same connection. Default: False");
1998 System.err.println(" presplit Create presplit table. If a table with same name exists,"
1999 + " it'll be deleted and recreated (instead of verifying count of its existing regions). "
2000 + "Recommended for accurate perf analysis (see guide). Default: disabled");
2001 System.err.println(" inmemory Tries to keep the HFiles of the CF " +
2002 "inmemory as far as possible. Not guaranteed that reads are always served " +
2003 "from memory. Default: false");
2004 System.err.println(" usetags Writes tags along with KVs. Use with HFile V3. " +
2005 "Default: false");
2006 System.err.println(" numoftags Specify the no of tags that would be needed. " +
2007 "This works only if usetags is true. Default: " + DEFAULT_OPTS.noOfTags);
2008 System.err.println(" filterAll Helps to filter out all the rows on the server side" +
2009 " there by not returning any thing back to the client. Helps to check the server side" +
2010 " performance. Uses FilterAllFilter internally. ");
2011 System.err.println(" latency Set to report operation latencies. Default: False");
2012 System.err.println(" bloomFilter Bloom filter type, one of " + Arrays.toString(BloomType.values()));
2013 System.err.println(" blockEncoding Block encoding to use. Value should be one of "
2014 + Arrays.toString(DataBlockEncoding.values()) + ". Default: NONE");
2015 System.err.println(" valueSize Pass value size to use: Default: " +
2016 DEFAULT_OPTS.getValueSize());
2017 System.err.println(" valueRandom Set if we should vary value size between 0 and " +
2018 "'valueSize'; set on read for stats on size: Default: Not set.");
2019 System.err.println(" valueZipf Set if we should vary value size between 0 and " +
2020 "'valueSize' in zipf form: Default: Not set.");
2021 System.err.println(" period Report every 'period' rows: " +
2022 "Default: opts.perClientRunRows / 10 = " + DEFAULT_OPTS.getPerClientRunRows()/10);
2023 System.err.println(" multiGet Batch gets together into groups of N. Only supported " +
2024 "by randomRead. Default: disabled");
2025 System.err.println(" blockSize Blocksize to use when writing out hfiles. ");
2026 System.err.println(" addColumns Adds columns to scans/gets explicitly. Default: true");
2027 System.err.println(" replicas Enable region replica testing. Defaults: 1.");
2028 System.err.println(" splitPolicy Specify a custom RegionSplitPolicy for the table.");
2029 System.err.println(" randomSleep Do a random sleep before each get between 0 and entered value. Defaults: 0");
2030 System.err.println(" columns Columns to write per row. Default: 1");
2031 System.err.println(" families Specify number of column families for the table. Default: 1");
2032 System.err.println(" caching Scan caching to use. Default: 30");
2033 System.err.println(" bufferSize Set the value of client side buffering. Default: 2MB");
2034 System.err.println();
2035 System.err.println(" Note: -D properties will be applied to the conf used. ");
2036 System.err.println(" For example: ");
2037 System.err.println(" -Dmapreduce.output.fileoutputformat.compress=true");
2038 System.err.println(" -Dmapreduce.task.timeout=60000");
2039 System.err.println();
2040 System.err.println("Command:");
2041 for (CmdDescriptor command : COMMANDS.values()) {
2042 System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
2043 }
2044 System.err.println();
2045 System.err.println("Args:");
2046 System.err.println(" nclients Integer. Required. Total number of clients "
2047 + "(and HRegionServers) running. 1 <= value <= 500");
2048 System.err.println("Examples:");
2049 System.err.println(" To run a single client doing the default 1M sequentialWrites:");
2050 System.err.println(" $ bin/hbase " + className + " sequentialWrite 1");
2051 System.err.println(" To run 10 clients doing increments over ten rows:");
2052 System.err.println(" $ bin/hbase " + className + " --rows=10 --nomapred increment 10");
2053 }
2054
2055
2056
2057
2058
2059
2060
2061 static TestOptions parseOpts(Queue<String> args) {
2062 TestOptions opts = new TestOptions();
2063
2064 String cmd = null;
2065 while ((cmd = args.poll()) != null) {
2066 if (cmd.equals("-h") || cmd.startsWith("--h")) {
2067
2068 args.add(cmd);
2069 break;
2070 }
2071
2072 final String nmr = "--nomapred";
2073 if (cmd.startsWith(nmr)) {
2074 opts.nomapred = true;
2075 continue;
2076 }
2077
2078 final String rows = "--rows=";
2079 if (cmd.startsWith(rows)) {
2080 opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length()));
2081 continue;
2082 }
2083
2084 final String sampleRate = "--sampleRate=";
2085 if (cmd.startsWith(sampleRate)) {
2086 opts.sampleRate = Float.parseFloat(cmd.substring(sampleRate.length()));
2087 continue;
2088 }
2089
2090 final String table = "--table=";
2091 if (cmd.startsWith(table)) {
2092 opts.tableName = cmd.substring(table.length());
2093 continue;
2094 }
2095
2096 final String startRow = "--startRow=";
2097 if (cmd.startsWith(startRow)) {
2098 opts.startRow = Integer.parseInt(cmd.substring(startRow.length()));
2099 continue;
2100 }
2101
2102 final String compress = "--compress=";
2103 if (cmd.startsWith(compress)) {
2104 opts.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
2105 continue;
2106 }
2107
2108 final String traceRate = "--traceRate=";
2109 if (cmd.startsWith(traceRate)) {
2110 opts.traceRate = Double.parseDouble(cmd.substring(traceRate.length()));
2111 continue;
2112 }
2113
2114 final String blockEncoding = "--blockEncoding=";
2115 if (cmd.startsWith(blockEncoding)) {
2116 opts.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
2117 continue;
2118 }
2119
2120 final String flushCommits = "--flushCommits=";
2121 if (cmd.startsWith(flushCommits)) {
2122 opts.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
2123 continue;
2124 }
2125
2126 final String writeToWAL = "--writeToWAL=";
2127 if (cmd.startsWith(writeToWAL)) {
2128 opts.writeToWAL = Boolean.parseBoolean(cmd.substring(writeToWAL.length()));
2129 continue;
2130 }
2131
2132 final String presplit = "--presplit=";
2133 if (cmd.startsWith(presplit)) {
2134 opts.presplitRegions = Integer.parseInt(cmd.substring(presplit.length()));
2135 continue;
2136 }
2137
2138 final String inMemory = "--inmemory=";
2139 if (cmd.startsWith(inMemory)) {
2140 opts.inMemoryCF = Boolean.parseBoolean(cmd.substring(inMemory.length()));
2141 continue;
2142 }
2143
2144 final String autoFlush = "--autoFlush=";
2145 if (cmd.startsWith(autoFlush)) {
2146 opts.autoFlush = Boolean.parseBoolean(cmd.substring(autoFlush.length()));
2147 continue;
2148 }
2149
2150 final String onceCon = "--oneCon=";
2151 if (cmd.startsWith(onceCon)) {
2152 opts.oneCon = Boolean.parseBoolean(cmd.substring(onceCon.length()));
2153 continue;
2154 }
2155
2156 final String latency = "--latency";
2157 if (cmd.startsWith(latency)) {
2158 opts.reportLatency = true;
2159 continue;
2160 }
2161
2162 final String multiGet = "--multiGet=";
2163 if (cmd.startsWith(multiGet)) {
2164 opts.multiGet = Integer.parseInt(cmd.substring(multiGet.length()));
2165 continue;
2166 }
2167
2168 final String useTags = "--usetags=";
2169 if (cmd.startsWith(useTags)) {
2170 opts.useTags = Boolean.parseBoolean(cmd.substring(useTags.length()));
2171 continue;
2172 }
2173
2174 final String noOfTags = "--numoftags=";
2175 if (cmd.startsWith(noOfTags)) {
2176 opts.noOfTags = Integer.parseInt(cmd.substring(noOfTags.length()));
2177 continue;
2178 }
2179
2180 final String replicas = "--replicas=";
2181 if (cmd.startsWith(replicas)) {
2182 opts.replicas = Integer.parseInt(cmd.substring(replicas.length()));
2183 continue;
2184 }
2185
2186 final String filterOutAll = "--filterAll";
2187 if (cmd.startsWith(filterOutAll)) {
2188 opts.filterAll = true;
2189 continue;
2190 }
2191
2192 final String size = "--size=";
2193 if (cmd.startsWith(size)) {
2194 opts.size = Float.parseFloat(cmd.substring(size.length()));
2195 continue;
2196 }
2197
2198 final String splitPolicy = "--splitPolicy=";
2199 if (cmd.startsWith(splitPolicy)) {
2200 opts.splitPolicy = cmd.substring(splitPolicy.length());
2201 continue;
2202 }
2203
2204 final String randomSleep = "--randomSleep=";
2205 if (cmd.startsWith(randomSleep)) {
2206 opts.randomSleep = Integer.parseInt(cmd.substring(randomSleep.length()));
2207 continue;
2208 }
2209
2210 final String bloomFilter = "--bloomFilter=";
2211 if (cmd.startsWith(bloomFilter)) {
2212 opts.bloomType = BloomType.valueOf(cmd.substring(bloomFilter.length()));
2213 continue;
2214 }
2215
2216 final String blockSize = "--blockSize=";
2217 if(cmd.startsWith(blockSize) ) {
2218 opts.blockSize = Integer.parseInt(cmd.substring(blockSize.length()));
2219 continue;
2220 }
2221
2222 final String valueSize = "--valueSize=";
2223 if (cmd.startsWith(valueSize)) {
2224 opts.valueSize = Integer.parseInt(cmd.substring(valueSize.length()));
2225 continue;
2226 }
2227
2228 final String valueRandom = "--valueRandom";
2229 if (cmd.startsWith(valueRandom)) {
2230 opts.valueRandom = true;
2231 if (opts.valueZipf) {
2232 throw new IllegalStateException("Either valueZipf or valueRandom but not both");
2233 }
2234 continue;
2235 }
2236
2237 final String valueZipf = "--valueZipf";
2238 if (cmd.startsWith(valueZipf)) {
2239 opts.valueZipf = true;
2240 if (opts.valueRandom) {
2241 throw new IllegalStateException("Either valueZipf or valueRandom but not both");
2242 }
2243 continue;
2244 }
2245
2246 final String period = "--period=";
2247 if (cmd.startsWith(period)) {
2248 opts.period = Integer.parseInt(cmd.substring(period.length()));
2249 continue;
2250 }
2251
2252 final String addColumns = "--addColumns=";
2253 if (cmd.startsWith(addColumns)) {
2254 opts.addColumns = Boolean.parseBoolean(cmd.substring(addColumns.length()));
2255 continue;
2256 }
2257
2258 final String columns = "--columns=";
2259 if (cmd.startsWith(columns)) {
2260 opts.columns = Integer.parseInt(cmd.substring(columns.length()));
2261 continue;
2262 }
2263
2264 final String families = "--families=";
2265 if (cmd.startsWith(families)) {
2266 opts.families = Integer.parseInt(cmd.substring(families.length()));
2267 continue;
2268 }
2269
2270 final String caching = "--caching=";
2271 if (cmd.startsWith(caching)) {
2272 opts.caching = Integer.parseInt(cmd.substring(caching.length()));
2273 continue;
2274 }
2275
2276 final String bufferSize = "--bufferSize=";
2277 if (cmd.startsWith(bufferSize)) {
2278 opts.bufferSize = Long.parseLong(cmd.substring(bufferSize.length()));
2279 continue;
2280 }
2281
2282 if (isCommandClass(cmd)) {
2283 opts.cmdName = cmd;
2284 try {
2285 opts.numClientThreads = Integer.parseInt(args.remove());
2286 } catch (NoSuchElementException | NumberFormatException e) {
2287 throw new IllegalArgumentException("Command " + cmd + " does not have threads number", e);
2288 }
2289 int rowsPerGB = getRowsPerGB(opts);
2290 if (opts.size != DEFAULT_OPTS.size &&
2291 opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
2292 throw new IllegalArgumentException(rows + " and " + size + " are mutually exclusive arguments.");
2293 }
2294 if (opts.size != DEFAULT_OPTS.size) {
2295
2296 opts.totalRows = (int) opts.size * rowsPerGB;
2297 opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
2298 } else {
2299 opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
2300 opts.size = opts.totalRows / rowsPerGB;
2301 }
2302 break;
2303 } else {
2304 printUsageAndExit("ERROR: Unrecognized option/command: " + cmd, -1);
2305 }
2306
2307
2308 System.err.println("Error: Wrong option or command: " + cmd);
2309 args.add(cmd);
2310 break;
2311 }
2312 return opts;
2313 }
2314
2315 static int getRowsPerGB(final TestOptions opts) {
2316 return ONE_GB / ((opts.valueRandom? opts.valueSize/2: opts.valueSize) * opts.getColumns() *
2317 opts.getFamilies());
2318 }
2319
2320 @Override
2321 public int run(String[] args) throws Exception {
2322
2323
2324 int errCode = -1;
2325 if (args.length < 1) {
2326 printUsage();
2327 return errCode;
2328 }
2329
2330 try {
2331 LinkedList<String> argv = new LinkedList<String>();
2332 argv.addAll(Arrays.asList(args));
2333 TestOptions opts = parseOpts(argv);
2334
2335
2336 if (!argv.isEmpty()) {
2337 errCode = 0;
2338 printUsage();
2339 return errCode;
2340 }
2341
2342
2343 if (opts.numClientThreads <= 0) {
2344 throw new IllegalArgumentException("Number of clients must be > 0");
2345 }
2346
2347
2348 if (opts.cmdName == null) {
2349 printUsage();
2350 return errCode;
2351 }
2352
2353 Class<? extends Test> cmdClass = determineCommandClass(opts.cmdName);
2354 if (cmdClass != null) {
2355 runTest(cmdClass, opts);
2356 errCode = 0;
2357 }
2358
2359 } catch (Exception e) {
2360 e.printStackTrace();
2361 }
2362
2363 return errCode;
2364 }
2365
2366 private static boolean isCommandClass(String cmd) {
2367 return COMMANDS.containsKey(cmd);
2368 }
2369
2370 private static Class<? extends Test> determineCommandClass(String cmd) {
2371 CmdDescriptor descriptor = COMMANDS.get(cmd);
2372 return descriptor != null ? descriptor.getCmdClass() : null;
2373 }
2374
2375 public static void main(final String[] args) throws Exception {
2376 int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
2377 System.exit(res);
2378 }
2379 }