/**
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
    * with the License.  You may obtain a copy of the License at
    *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.hadoop.hbase.test;
  
  import java.io.IOException;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Set;
  import java.util.concurrent.Executors;
  import java.util.concurrent.ScheduledExecutorService;
  import java.util.concurrent.ScheduledFuture;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
  import org.apache.commons.lang.math.RandomUtils;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.hbase.HBaseConfiguration;
  import org.apache.hadoop.hbase.HRegionLocation;
  import org.apache.hadoop.hbase.IntegrationTestIngest;
  import org.apache.hadoop.hbase.IntegrationTestingUtility;
  import org.apache.hadoop.hbase.RegionLocations;
  import org.apache.hadoop.hbase.testclassification.IntegrationTests;
  import org.apache.hadoop.hbase.TableName;
  import org.apache.hadoop.hbase.chaos.factories.MonkeyFactory;
  import org.apache.hadoop.hbase.client.Admin;
  import org.apache.hadoop.hbase.client.ClusterConnection;
  import org.apache.hadoop.hbase.client.Consistency;
  import org.apache.hadoop.hbase.client.Get;
  import org.apache.hadoop.hbase.client.Result;
  import org.apache.hadoop.hbase.client.Table;
  import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
  import org.apache.hadoop.hbase.util.LoadTestTool;
  import org.apache.hadoop.hbase.util.MultiThreadedReader;
  import org.apache.hadoop.hbase.util.Threads;
  import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator;
  import org.apache.hadoop.util.StringUtils;
  import org.apache.hadoop.util.ToolRunner;
  import org.junit.Assert;
  import org.junit.experimental.categories.Category;
  
  import com.google.common.collect.Lists;
  
  /**
   * An IntegrationTest for doing reads with a timeout, to a read-only table with region
   * replicas. ChaosMonkey is run which kills the region servers and master, but ensures
   * that meta region server is not killed, and at most 2 region servers are dead at any point
   * in time. The expected behavior is that all reads with stale mode true will return
   * before the timeout (5 sec by default). The test fails if the read requests does not finish
   * in time.
   *
   * <p> This test uses LoadTestTool to read and write the data from a single client but
   * multiple threads. The data is written first, then we allow the region replicas to catch
   * up. Then we start the reader threads doing get requests with stale mode true. Chaos Monkey is
   * started after some delay (20 sec by default) after the reader threads are started so that
   * there is enough time to fully cache meta.
   *
   * These parameters (and some other parameters from LoadTestTool) can be used to
   * control behavior, given values are default:
   * <pre>
   * -Dhbase.IntegrationTestTimeBoundedRequestsWithRegionReplicas.runtime=600000
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_regions_per_server=5
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.get_timeout_ms=5000
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_keys_per_server=2500
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.region_replication=3
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_read_threads=20
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_write_threads=20
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_regions_per_server=5
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.chaos_monkey_delay=20000
   * </pre>
   * Use this test with "serverKilling" ChaosMonkey. Sample usage:
   * <pre>
   * hbase org.apache.hadoop.hbase.test.IntegrationTestTimeBoundedRequestsWithRegionReplicas
   * -Dhbase.IntegrationTestTimeBoundedRequestsWithRegionReplicas.runtime=600000
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_write_threads=40
   * -DIntegrationTestTimeBoundedRequestsWithRegionReplicas.num_read_threads=40
   * -Dhbase.ipc.client.allowsInterrupt=true --monkey serverKilling
   * </pre>
   */
  @Category(IntegrationTests.class)
  public class IntegrationTestTimeBoundedRequestsWithRegionReplicas extends IntegrationTestIngest {
  
   private static final Log LOG = LogFactory.getLog(
     IntegrationTestTimeBoundedRequestsWithRegionReplicas.class);
 
   private static final String TEST_NAME
     = IntegrationTestTimeBoundedRequestsWithRegionReplicas.class.getSimpleName();
 
   protected static final long DEFAULT_GET_TIMEOUT = 5000; // 5 sec
   protected static final String GET_TIMEOUT_KEY = "get_timeout_ms";
 
   protected static final long DEFAUL_CHAOS_MONKEY_DELAY = 20 * 1000; // 20 sec
   protected static final String CHAOS_MONKEY_DELAY_KEY = "chaos_monkey_delay";
 
   protected static final int DEFAULT_REGION_REPLICATION = 3;
 
   @Override
   protected void startMonkey() throws Exception {
     // we do not want to start the monkey at the start of the test.
   }
 
   @Override
   protected MonkeyFactory getDefaultMonkeyFactory() {
     return MonkeyFactory.getFactory(MonkeyFactory.CALM);
   }
 
   @Override
   public void setConf(Configuration conf) {
     super.setConf(conf);
     // default replication for this test is 3
     String clazz = this.getClass().getSimpleName();
     conf.setIfUnset(String.format("%s.%s", clazz, LoadTestTool.OPT_REGION_REPLICATION),
       Integer.toString(DEFAULT_REGION_REPLICATION));
   }
 
   protected void writeData(int colsPerKey, int recordSize, int writeThreads,
       long startKey, long numKeys) throws IOException {
     int ret = loadTool.run(getArgsForLoadTestTool("-write",
       String.format("%d:%d:%d", colsPerKey, recordSize, writeThreads), startKey, numKeys));
     if (0 != ret) {
       String errorMsg = "Load failed with error code " + ret;
       LOG.error(errorMsg);
       Assert.fail(errorMsg);
     }
   }
 
   @Override
   protected void runIngestTest(long defaultRunTime, long keysPerServerPerIter, int colsPerKey,
       int recordSize, int writeThreads, int readThreads) throws Exception {
     LOG.info("Cluster size:"+
       util.getHBaseClusterInterface().getClusterStatus().getServersSize());
 
     long start = System.currentTimeMillis();
     String runtimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName());
     long runtime = util.getConfiguration().getLong(runtimeKey, defaultRunTime);
     long startKey = 0;
 
     long numKeys = getNumKeys(keysPerServerPerIter);
 
 
     // write data once
     LOG.info("Writing some data to the table");
     writeData(colsPerKey, recordSize, writeThreads, startKey, numKeys);
 
     // flush the table
     LOG.info("Flushing the table");
     Admin admin = util.getHBaseAdmin();
     admin.flush(getTablename());
 
     // re-open the regions to make sure that the replicas are up to date
     long refreshTime = conf.getLong(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
     if (refreshTime > 0 && refreshTime <= 10000) {
       LOG.info("Sleeping " + refreshTime + "ms to ensure that the data is replicated");
       Threads.sleep(refreshTime*3);
     } else {
       LOG.info("Reopening the table");
       admin.disableTable(getTablename());
       admin.enableTable(getTablename());
     }
 
     // We should only start the ChaosMonkey after the readers are started and have cached
     // all of the region locations. Because the meta is not replicated, the timebounded reads
     // will timeout if meta server is killed.
     // We will start the chaos monkey after 1 minute, and since the readers are reading random
     // keys, it should be enough to cache every region entry.
     long chaosMonkeyDelay = conf.getLong(String.format("%s.%s", TEST_NAME, CHAOS_MONKEY_DELAY_KEY)
       , DEFAUL_CHAOS_MONKEY_DELAY);
     ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
     LOG.info(String.format("ChaosMonkey delay is : %d seconds. Will start %s " +
         "ChaosMonkey after delay", chaosMonkeyDelay / 1000, monkeyToUse));
     ScheduledFuture<?> result = executorService.schedule(new Runnable() {
       @Override
       public void run() {
         try {
           LOG.info("Starting ChaosMonkey");
           monkey.start();
           monkey.waitForStop();
         } catch (Exception e) {
           LOG.warn(StringUtils.stringifyException(e));
         }
 
       }
     }, chaosMonkeyDelay, TimeUnit.MILLISECONDS);
 
     // set the intended run time for the reader. The reader will do read requests
     // to random keys for this amount of time.
     long remainingTime = runtime - (System.currentTimeMillis() - start);
     if (remainingTime <= 0) {
       LOG.error("The amount of time left for the test to perform random reads is "
           + "non-positive. Increase the test execution time via "
           + String.format(RUN_TIME_KEY,
                 IntegrationTestTimeBoundedRequestsWithRegionReplicas.class.getSimpleName())
           + " or reduce the amount of data written per server via "
           + IntegrationTestTimeBoundedRequestsWithRegionReplicas.class.getSimpleName()
           + "." + IntegrationTestIngest.NUM_KEYS_PER_SERVER_KEY);
       throw new IllegalArgumentException("No time remains to execute random reads");
     }
     LOG.info("Reading random keys from the table for " + remainingTime/60000 + " min");
     this.conf.setLong(
       String.format(RUN_TIME_KEY, TimeBoundedMultiThreadedReader.class.getSimpleName())
       , remainingTime); // load tool shares the same conf
 
     // now start the readers which will run for configured run time
     try {
       int ret = loadTool.run(getArgsForLoadTestTool("-read", String.format("100:%d", readThreads)
         , startKey, numKeys));
       if (0 != ret) {
         String errorMsg = "Verification failed with error code " + ret;
         LOG.error(errorMsg);
         Assert.fail(errorMsg);
       }
     } finally {
       if (result != null) result.cancel(false);
       monkey.stop("Stopping the test");
       monkey.waitForStop();
       executorService.shutdown();
     }
   }
 
   @Override
   protected String[] getArgsForLoadTestTool(String mode, String modeSpecificArg, long startKey,
       long numKeys) {
     List<String> args = Lists.newArrayList(super.getArgsForLoadTestTool(
       mode, modeSpecificArg, startKey, numKeys));
     args.add("-reader");
     args.add(TimeBoundedMultiThreadedReader.class.getName());
     return args.toArray(new String[args.size()]);
   }
 
   public static class TimeBoundedMultiThreadedReader extends MultiThreadedReader {
     protected long timeoutNano;
     protected AtomicLong timedOutReads = new AtomicLong();
     protected long runTime;
     protected Thread timeoutThread;
     protected AtomicLong staleReads = new AtomicLong();
 
     public TimeBoundedMultiThreadedReader(LoadTestDataGenerator dataGen, Configuration conf,
         TableName tableName, double verifyPercent) throws IOException {
       super(dataGen, conf, tableName, verifyPercent);
       long timeoutMs = conf.getLong(
         String.format("%s.%s", TEST_NAME, GET_TIMEOUT_KEY), DEFAULT_GET_TIMEOUT);
       timeoutNano = timeoutMs * 1000000;
       LOG.info("Timeout for gets: " + timeoutMs);
       String runTimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName());
       this.runTime = conf.getLong(runTimeKey, -1);
       if (this.runTime <= 0) {
         throw new IllegalArgumentException("Please configure " + runTimeKey);
       }
     }
 
     @Override
     public void waitForFinish() {
       try {
         this.timeoutThread.join();
       } catch (InterruptedException e) {
         e.printStackTrace();
       }
       this.aborted = true;
       super.waitForFinish();
     }
 
     @Override
     protected String progressInfo() {
       StringBuilder builder = new StringBuilder(super.progressInfo());
       appendToStatus(builder, "stale_reads", staleReads.get());
       appendToStatus(builder, "get_timeouts", timedOutReads.get());
       return builder.toString();
     }
 
     @Override
     public void start(long startKey, long endKey, int numThreads) throws IOException {
       super.start(startKey, endKey, numThreads);
       this.timeoutThread = new TimeoutThread(this.runTime);
       this.timeoutThread.start();
     }
 
     @Override
     protected HBaseReaderThread createReaderThread(int readerId) throws IOException {
       return new TimeBoundedMultiThreadedReaderThread(readerId);
     }
 
     private class TimeoutThread extends Thread {
       long timeout;
       long reportInterval = 60000;
       public TimeoutThread(long timeout) {
         this.timeout = timeout;
       }
 
       @Override
       public void run() {
         while (true) {
           long rem = Math.min(timeout, reportInterval);
           if (rem <= 0) {
             break;
           }
           LOG.info("Remaining execution time:" + timeout / 60000 + " min");
           Threads.sleep(rem);
           timeout -= rem;
         }
       }
     }
 
     public class TimeBoundedMultiThreadedReaderThread
       extends MultiThreadedReader.HBaseReaderThread {
 
       public TimeBoundedMultiThreadedReaderThread(int readerId) throws IOException {
         super(readerId);
       }
 
       @Override
       protected Get createGet(long keyToRead) throws IOException {
         Get get = super.createGet(keyToRead);
         get.setConsistency(Consistency.TIMELINE);
         return get;
       }
 
       @Override
       protected long getNextKeyToRead() {
         // always read a random key, assuming that the writer has finished writing all keys
         long key = startKey + Math.abs(RandomUtils.nextLong())
             % (endKey - startKey);
         return key;
       }
 
       @Override
       protected void verifyResultsAndUpdateMetrics(boolean verify, Get[] gets, long elapsedNano,
           Result[] results, Table table, boolean isNullExpected)
           throws IOException {
         super.verifyResultsAndUpdateMetrics(verify, gets, elapsedNano, results, table, isNullExpected);
         for (Result r : results) {
           if (r.isStale()) staleReads.incrementAndGet();
         }
         // we actually do not timeout and cancel the reads after timeout. We just wait for the RPC
         // to complete, but if the request took longer than timeout, we treat that as error.
         if (elapsedNano > timeoutNano) {
           timedOutReads.incrementAndGet();
           numReadFailures.addAndGet(1); // fail the test
           for (Result r : results) {
             LOG.error("FAILED FOR " + r);
             RegionLocations rl = ((ClusterConnection)connection).
                 locateRegion(tableName, r.getRow(), true, true);
             HRegionLocation locations[] = rl.getRegionLocations();
             for (HRegionLocation h : locations) {
               LOG.error("LOCATION " + h);
             }
           }
         }
       }
     }
   }
 
   public static void main(String[] args) throws Exception {
     Configuration conf = HBaseConfiguration.create();
     IntegrationTestingUtility.setUseDistributedCluster(conf);
     int ret = ToolRunner.run(conf, new IntegrationTestTimeBoundedRequestsWithRegionReplicas(), args);
     System.exit(ret);
   }
 }