1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.procedure2.store.wal;
20
21 import java.io.IOException;
22 import java.util.concurrent.Callable;
23 import java.util.concurrent.ExecutorService;
24 import java.util.concurrent.Executors;
25 import java.util.concurrent.Future;
26 import java.util.concurrent.TimeUnit;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicLong;
29
30 import org.apache.commons.cli.CommandLine;
31 import org.apache.commons.cli.Option;
32 import org.apache.hadoop.fs.*;
33 import org.apache.hadoop.conf.*;
34 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
35 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
36 import org.apache.hadoop.hbase.procedure2.util.*;
37
38 import org.apache.hadoop.hbase.util.AbstractHBaseTool;
39
40 public class ProcedureWALPerformanceEvaluation extends AbstractHBaseTool {
41 protected static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility();
42
43
44 public static int DEFAULT_NUM_THREADS = 20;
45 public static Option NUM_THREADS_OPTION = new Option("threads", true,
46 "Number of parallel threads which will write insert/updates/deletes to WAL. Default: "
47 + DEFAULT_NUM_THREADS);
48 public static int DEFAULT_NUM_PROCS = 1000000;
49 public static Option NUM_PROCS_OPTION = new Option("procs", true,
50 "Total number of procedures. Each procedure writes one insert and one update. Default: "
51 + DEFAULT_NUM_PROCS);
52 public static int DEFAULT_NUM_WALS = 0;
53 public static Option NUM_WALS_OPTION = new Option("wals", true,
54 "Number of WALs to write. If -ve or 0, uses " + WALProcedureStore.ROLL_THRESHOLD_CONF_KEY +
55 " conf to roll the logs. Default: " + DEFAULT_NUM_WALS);
56 public static int DEFAULT_STATE_SIZE = 1024;
57 public static Option STATE_SIZE_OPTION = new Option("size", true,
58 "Size of serialized state in bytes to write on update. Default: " + DEFAULT_STATE_SIZE
59 + "bytes");
60 public static Option SYNC_OPTION = new Option("sync", true,
61 "Type of sync to use when writing WAL contents to file system. Accepted values: hflush, "
62 + "hsync, nosync. Default: hflush");
63 public static String DEFAULT_SYNC_OPTION = "hflush";
64
65 public int numThreads;
66 public long numProcs;
67 public long numProcsPerWal = Long.MAX_VALUE;
68 public int numWals;
69 public String syncType;
70 public int stateSize;
71 static byte[] serializedState;
72 private WALProcedureStore store;
73
74
75 private AtomicLong procIds = new AtomicLong(0);
76 private AtomicBoolean workersFailed = new AtomicBoolean(false);
77
78 private static final int WORKER_THREADS_TIMEOUT_SEC = 600;
79
80
81 private void setupConf() {
82 conf.setBoolean(WALProcedureStore.USE_HSYNC_CONF_KEY, "hsync".equals(syncType));
83 if (numWals > 0) {
84 conf.setLong(WALProcedureStore.ROLL_THRESHOLD_CONF_KEY, Long.MAX_VALUE);
85 numProcsPerWal = numProcs / numWals;
86 }
87 }
88
89 private void setupProcedureStore() throws IOException {
90 Path testDir = UTIL.getDataTestDir();
91 FileSystem fs = testDir.getFileSystem(conf);
92 Path logDir = new Path(testDir, "proc-logs");
93 System.out.println("Logs directory : " + logDir.toString());
94 fs.delete(logDir, true);
95 if ("nosync".equals(syncType)) {
96 store = new NoSyncWalProcedureStore(conf, logDir);
97 } else {
98 store = ProcedureTestingUtility.createWalStore(conf, logDir);
99 }
100 store.start(numThreads);
101 store.recoverLease();
102 store.load(new ProcedureTestingUtility.LoadCounter());
103 System.out.println("Starting new log : "
104 + store.getActiveLogs().get(store.getActiveLogs().size() - 1));
105 }
106
107 private void tearDownProcedureStore() {
108 store.stop(false);
109 try {
110 store.getFileSystem().delete(store.getWALDir(), true);
111 } catch (IOException e) {
112 System.err.println("Error: Couldn't delete log dir. You can delete it manually to free up "
113 + "disk space. Location: " + store.getWALDir().toString());
114 e.printStackTrace();
115 }
116 }
117
118
119
120
121 @Override
122 public void processOptions(CommandLine cmd) {
123 numThreads = getOptionAsInt(cmd, NUM_THREADS_OPTION.getOpt(), DEFAULT_NUM_THREADS);
124 numProcs = getOptionAsInt(cmd, NUM_PROCS_OPTION.getOpt(), DEFAULT_NUM_PROCS);
125 numWals = getOptionAsInt(cmd, NUM_WALS_OPTION.getOpt(), DEFAULT_NUM_WALS);
126 syncType = cmd.getOptionValue(SYNC_OPTION.getOpt(), DEFAULT_SYNC_OPTION);
127 assert "hsync".equals(syncType) || "hflush".equals(syncType) || "nosync".equals(syncType):
128 "sync argument can only accept one of these three values: hsync, hflush, nosync";
129 stateSize = getOptionAsInt(cmd, STATE_SIZE_OPTION.getOpt(), DEFAULT_STATE_SIZE);
130 serializedState = new byte[stateSize];
131 setupConf();
132 }
133
134 @Override
135 public void addOptions() {
136 addOption(NUM_THREADS_OPTION);
137 addOption(NUM_PROCS_OPTION);
138 addOption(NUM_WALS_OPTION);
139 addOption(SYNC_OPTION);
140 addOption(STATE_SIZE_OPTION);
141 }
142
143 @Override
144 public int doWork() {
145 try {
146 setupProcedureStore();
147 ExecutorService executor = Executors.newFixedThreadPool(numThreads);
148 Future<?>[] futures = new Future<?>[numThreads];
149
150 long start = System.currentTimeMillis();
151 for (int i = 0; i < numThreads; i++) {
152 futures[i] = executor.submit(this.new Worker(start));
153 }
154 boolean failure = false;
155 try {
156 for (Future<?> future : futures) {
157 long timeout = start + WORKER_THREADS_TIMEOUT_SEC * 1000 - System.currentTimeMillis();
158 failure |= (future.get(timeout, TimeUnit.MILLISECONDS).equals(EXIT_FAILURE));
159 }
160 } catch (Exception e) {
161 System.err.println("Exception in worker thread.");
162 e.printStackTrace();
163 return EXIT_FAILURE;
164 }
165 executor.shutdown();
166 if (failure) {
167 return EXIT_FAILURE;
168 }
169 long timeTaken = System.currentTimeMillis() - start;
170 System.out.println("******************************************");
171 System.out.println("Num threads : " + numThreads);
172 System.out.println("Num procedures : " + numProcs);
173 System.out.println("Sync type : " + syncType);
174 System.out.println("Time taken : " + (timeTaken / 1000.0f) + "sec");
175 System.out.println("******************************************");
176 return EXIT_SUCCESS;
177 } catch (IOException e) {
178 e.printStackTrace();
179 return EXIT_FAILURE;
180 } finally {
181 tearDownProcedureStore();
182 }
183 }
184
185
186
187
188
189
190
191
192
193
194 class Worker implements Callable<Integer> {
195 final long start;
196
197 public Worker(long start) {
198 this.start = start;
199 }
200
201
202 @Override
203 public Integer call() throws IOException {
204 while (true) {
205 if (workersFailed.get()) {
206 return EXIT_FAILURE;
207 }
208 long procId = procIds.getAndIncrement();
209 if (procId >= numProcs) {
210 break;
211 }
212 if (procId != 0 && procId % 10000 == 0) {
213 long ms = System.currentTimeMillis() - start;
214 System.out.println("Wrote " + procId + " procedures in "
215 + StringUtils.humanTimeDiff(ms));
216 }
217 try{
218 if (procId > 0 && procId % numProcsPerWal == 0) {
219 store.rollWriterForTesting();
220 System.out.println("Starting new log : "
221 + store.getActiveLogs().get(store.getActiveLogs().size() - 1));
222 }
223 } catch (IOException ioe) {
224
225 workersFailed.set(true);
226 System.err.println("Exception when rolling log file. Current procId = " + procId);
227 ioe.printStackTrace();
228 return EXIT_FAILURE;
229 }
230 ProcedureTestingUtility.TestProcedure proc =
231 new ProcedureTestingUtility.TestProcedure(procId);
232 proc.setData(serializedState);
233 store.insert(proc, null);
234 store.update(proc);
235 }
236 return EXIT_SUCCESS;
237 }
238 }
239
240 private class NoSyncWalProcedureStore extends WALProcedureStore {
241 public NoSyncWalProcedureStore(final Configuration conf, final Path logDir) throws IOException {
242 super(conf, logDir, new WALProcedureStore.LeaseRecovery() {
243 @Override
244 public void recoverFileLease(FileSystem fs, Path path) throws IOException {
245
246 }
247 });
248 }
249
250 @Override
251 protected long syncSlots(FSDataOutputStream stream, ByteSlot[] slots, int offset, int count)
252 throws IOException {
253 long totalSynced = 0;
254 for (int i = 0; i < count; ++i) {
255 totalSynced += slots[offset + i].size();
256 }
257 return totalSynced;
258 }
259 }
260
261 public static void main(String[] args) throws IOException {
262 ProcedureWALPerformanceEvaluation tool = new ProcedureWALPerformanceEvaluation();
263 tool.setConf(UTIL.getConfiguration());
264 tool.run(args);
265 }
266 }