KEYCLOAK-5372 Add warm-up, test-time, ramp-down times

testsuite/performance/README.log-tool.md

@@ -7,7 +7,7 @@ Perform the usual test run:
 mvn verify -Pteardown
 mvn verify -Pprovision
 mvn verify -Pgenerate-data -Ddataset=100users -Dimport.workers=10 -DhashIterations=100
-mvn verify -Ptest -Ddataset=100users -DrunUsers=200 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DsteadyLoadPeriod=10
+mvn verify -Ptest -Ddataset=100users -DrunUsers=200 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DmeasurementPeriod=60
 ```
 
 Now analyze the generated simulation.log (adjust LOG_DIR, FROM, and TO):

testsuite/performance/README.md

@@ -27,7 +27,7 @@ mvn clean install
 # Make sure your Docker daemon is running THEN
 mvn verify -Pprovision
 mvn verify -Pgenerate-data -Ddataset=100u -DnumOfWorkers=10 -DhashIterations=100
-mvn verify -Ptest -Ddataset=100u -DrunUsers=200 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DsteadyLoadPeriod=10
+mvn verify -Ptest -Ddataset=100u -DrunUsers=200 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DmeasurementPeriod=60
 
 ```
 
@@ -143,7 +143,8 @@ Usage: `mvn verify -Ptest[,cluster] [-DtestParameter=value]`.
 | `dataset` | Name of the dataset to use. (Individual dataset properties can be overridden with `-Ddataset.property=value`.) | `default` |
 | `runUsers` | Number of users for the simulation run. | `1` |
 | `rampUpPeriod` | Period during which the users will be ramped up. (seconds) | `0` |
-| `steadyLoadPeriod` | A period of steady load. (seconds) | `30` |
+| `warmUpPeriod` | Period with steady number of users intended for the system under test to warm up. (seconds) | `0` |
+| `measurementPeriod` | A measurement period after the system is warmed up. (seconds) | `30` |
 | `rampDownASAP` | When `true` the test will be checking for ramp-down condition after each *scenario step*. When `false` the check will be done only at the end of a *scenario iteration*. | `false` |
 | `pace` | A dynamic pause after each *scenario iteration*. For example if the pace is 30s and one scenario iteration takes only 20s, the simulation will wait additional 10s before continuing to the next iteration. | `0` |
 | `userThinkTime` | Pause between individual scenario steps. | `5` |
@@ -159,7 +160,7 @@ Usage: `mvn verify -Ptest[,cluster] [-DtestParameter=value]`.
 
 Example:
 
-`mvn verify -Ptest -Dgatling.simulationClass=keycloak.AdminConsoleSimulation -Ddataset=100u -DrunUsers=1 -DsteadyLoadPeriod=30 -DuserThinkTime=0 -DrefreshTokenPeriod=15`
+`mvn verify -Ptest -Dgatling.simulationClass=keycloak.AdminConsoleSimulation -Ddataset=100u -DrunUsers=1 -DmeasurementPeriod=60 -DuserThinkTime=0 -DrefreshTokenPeriod=15`
 
 
 ## Monitoring

testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java

@@ -57,15 +57,18 @@ public class TestConfig {
     //
     public static final int runUsers = Integer.getInteger("runUsers", 1);
     public static final int rampUpPeriod = Integer.getInteger("rampUpPeriod", 0);
-    public static final int steadyLoadPeriod = Integer.getInteger("steadyLoadPeriod", 30);
+    public static final int warmUpPeriod = Integer.getInteger("warmUpPeriod", 0);
+    public static final int measurementPeriod = Integer.getInteger("measurementPeriod", 30);
     public static final boolean rampDownASAP = Boolean.getBoolean("rampDownASAP"); // check for rampdown condition after each scenario step
     public static final int pace = Integer.getInteger("pace", 0); // additional dynamic "pause buffer" between scenario loops
-    public static final int userThinkTime = Integer.getInteger("userThinkTime", 5);
-    public static final int refreshTokenPeriod = Integer.getInteger("refreshTokenPeriod", 10);
+    public static final int userThinkTime = Integer.getInteger("userThinkTime", 0);
+    public static final int refreshTokenPeriod = Integer.getInteger("refreshTokenPeriod", 0);
 
     // Computed timestamps
-    public static final long simulationStartTime = System.currentTimeMillis();//new Date().getTime();
-    public static final long rampDownPeriodStartTime = simulationStartTime + (rampUpPeriod + steadyLoadPeriod) * 1000;
+    public static final long simulationStartTime = System.currentTimeMillis();
+    public static final long warmUpStartTime = simulationStartTime + rampUpPeriod * 1000;
+    public static final long measurementStartTime = warmUpStartTime + warmUpPeriod * 1000;
+    public static final long measurementEndTime = measurementStartTime + measurementPeriod * 1000;
 
     //
     // Settings used by BasicOIDCSimulation to control behavior specific to BasicOIDCSimulation
@@ -114,15 +117,15 @@ public class TestConfig {
     }
 
     public static String toStringCommonTestParameters() {
-        return String.format(
-        "  runUsers: %s\n" + 
-        "  rampUpPeriod: %s\n"+ 
-        "  steadyLoadPeriod: %s\n"+
-        "  rampDownASAP: %s\n"+ 
-        "  pace: %s\n"+ 
-        "  userThinkTime: %s\n"+ 
-        "  refreshTokenPeriod: %s",
-                runUsers, rampUpPeriod, steadyLoadPeriod, rampDownASAP, pace, userThinkTime, refreshTokenPeriod
+        return String.format("  runUsers: %s\n"
+                + "  rampUpPeriod: %s\n"
+                + "  warmUpPeriod: %s\n"
+                + "  measurementPeriod: %s\n"
+                + "  rampDownASAP: %s\n"
+                + "  pace: %s\n"
+                + "  userThinkTime: %s\n"
+                + "  refreshTokenPeriod: %s",
+                runUsers, rampUpPeriod, warmUpPeriod, measurementPeriod, rampDownASAP, pace, userThinkTime, refreshTokenPeriod
         );
     }
 

testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala

@@ -3,6 +3,8 @@ package keycloak
 import io.gatling.core.Predef._
 import io.gatling.http.Predef._
 
+import keycloak.CommonScenarioBuilder._
+
 import io.gatling.core.validation.Validation
 import org.keycloak.performance.TestConfig
 
@@ -12,12 +14,6 @@ import org.keycloak.performance.TestConfig
   */
 class AdminConsoleSimulation extends Simulation {
 
-  def rampDownPeriodNotReached(): Validation[Boolean] = {
-    System.currentTimeMillis < TestConfig.rampDownPeriodStartTime
-  }
-
-
-
   println()
   println("Target server: " + TestConfig.serverUrisList.get(0))
   println()
@@ -95,7 +91,7 @@ class AdminConsoleSimulation extends Simulation {
 
 
   val adminScenario = scenario("AdminConsole")
-    .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) {
+    .asLongAs(s => rampDownNotStarted(), null, TestConfig.rampDownASAP) {
       pace(TestConfig.pace)
       adminSession.chainBuilder
     }

testsuite/performance/tests/src/test/scala/keycloak/BasicOIDCScenarioBuilder.scala

@@ -45,9 +45,6 @@ object BasicOIDCScenarioBuilder {
     missCounter.getAndDecrement() > 0
   }
 
-  def rampDownPeriodNotReached(): Validation[Boolean] = {
-    System.currentTimeMillis < TestConfig.rampDownPeriodStartTime
-  }
 }
 
 

testsuite/performance/tests/src/test/scala/keycloak/BasicOIDCSimulation.scala

@@ -2,6 +2,7 @@ package keycloak
 
 import io.gatling.core.Predef._
 import io.gatling.http.Predef._
+import keycloak.CommonScenarioBuilder._
 import keycloak.BasicOIDCScenarioBuilder._
 
 import org.keycloak.performance.TestConfig
@@ -49,7 +50,7 @@ class BasicOIDCSimulation extends Simulation {
 
 
   val usersScenario = scenario("users")
-    .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) {
+    .asLongAs(s => rampDownNotStarted(), null, TestConfig.rampDownASAP) {
       pace(TestConfig.pace)
       userSession.chainBuilder
     }

testsuite/performance/tests/src/test/scala/keycloak/CommonScenarioBuilder.scala

@@ -0,0 +1,31 @@
+package keycloak
+
+import io.gatling.core.Predef._
+import io.gatling.http.Predef._
+import org.keycloak.gatling.Predef._
+import keycloak.BasicOIDCScenarioBuilder._
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import io.gatling.core.pause.Normal
+import io.gatling.core.session.Session
+import io.gatling.core.structure.ChainBuilder
+import io.gatling.core.validation.Validation
+import org.jboss.perf.util.Util
+import org.jboss.perf.util.Util.randomUUID
+import org.keycloak.adapters.spi.HttpFacade.Cookie
+import org.keycloak.gatling.AuthorizeAction
+import org.keycloak.performance.TestConfig
+
+
+/**
+  * @author <a href="mailto:tkyjovsk@redhat.com">Tomas Kyjovsky</a>
+  */
+object CommonScenarioBuilder {
+
+  def rampDownNotStarted(): Validation[Boolean] = {
+    System.currentTimeMillis < TestConfig.measurementEndTime
+  }
+
+}
+