KEYCLOAK-5971 Replace "numOfIterations" parameter with "steadyLoadPeriod"

testsuite/performance/README.md

@@ -98,41 +98,68 @@ Dataset properties are loaded from `datasets/${dataset}.properties` file. Indivi
 
 Dataset data is first generated as a .json file, and then imported into Keycloak via Admin Client REST API.
 
+#### Dataset Properties
+
+| Property | Description | Value in the Default Dataset |
+| --- | --- | --- | 
+| `numOfRealms` | Number of realms to be created. | `1`  |
+| `usersPerRealm` | Number of users per realm. | `100`  |
+| `clientsPerRealm` | Number of clients per realm. | `2`  |
+| `realmRoles` | Number of realm-roles per realm. | `2`  |
+| `realmRolesPerUser` | Number of realm-roles assigned to a created user. Has to be less than or equal to `realmRoles`. | `2`  |
+| `clientRolesPerUser` | Number of client-roles assigned to a created user. Has to be less than or equal to `clientsPerRealm * clientRolesPerClient`. | `2`  |
+| `clientRolesPerClient` | Number of client-roles per created client. | `2`  |
+| `hashIterations` | Number of password hashing iterations. | `27500`  |
+
+
 #### Examples:
 - `mvn verify -Pgenerate-data` - generate default dataset
 - `mvn verify -Pgenerate-data -DusersPerRealm=5` - generate default dataset, override the `usersPerRealm` property
 - `mvn verify -Pgenerate-data -Ddataset=100u` - generate `100u` dataset
 - `mvn verify -Pgenerate-data -Ddataset=100r/default` - generate dataset based on `datasets/100r/default.properties`
 
-The data can also be exported from the database, and stored locally as `datasets/${dataset}.sql.gz`
-`DATASET=100u ./prepare-dump.sh`
+#### Export / Import Database Dump
 
 To speed up dataset initialization part, it is possible to pass `-Dexport-dump` option to have the generated dataset
 exported right after it has been generated. Then, if there is a data dump file available then `-Pimport-dump` 
 can be used to import the data directly into the database, bypassing Keycloak server completely.
 
-Usage: `mvn verify -Pimport-dump [-Ddataset=DATASET]`
+**Usage:** `mvn verify -Pimport-dump [-Ddataset=DATASET]`
 
-#### Example:
-- `mvn verify -Pimport-dump -Ddataset=100u` - import `datasets/100u.sql.gz` dump file created using `prepare-dump.sh`.
+**For example:**
+- `mvn verify -Pgenerate-data -Ddataset=100u -Dexport-dump` will generate data based on `datasets/100u.properties` and export a database dump to a file: `datasets/100u.sql.gz`.
+- `mvn verify -Pimport-dump -Ddataset=100u` will import the database dump from a file: `datasets/100u.sql.gz`, and reboot the server(s)
 
 
 ### Run Tests
 
-Usage: `mvn verify -Ptest [-DrunUsers=N] [-DrampUpPeriod=SECONDS] [-DnumOfIterations=N] [-Ddataset=DATASET] [-D<dataset.property>=<value>]* [-D<test.property>=<value>]* `.
+Usage: `mvn verify -Ptest[,cluster] [-DtestParameter=value]`.
 
-_*Note:* The same dataset properties which were used for data generation/import should be supplied to the `test` phase._
+#### Common Parameters
 
-The default test `keycloak.DefaultSimulation` takes the following additional properties:
+| Parameter | Description | Default Value |
+| --- | --- | --- | 
+| `gatling.simulationClass` | Classname of the simulation to be run. | `keycloak.DefaultSimulation`  |
+| `dataset` | Name of the dataset to use. (Individual dataset properties can be overridden with `-Ddataset.property=value`.) | `default` |
+| `runUsers` | Number of users for the simulation run. | `1` |
+| `rampUpPeriod` | Period during which the users will be ramped up. (seconds) | `0` |
+| `steadyLoadPeriod` | A period of steady load. (seconds) | `30` |
+| `rampDownASAP` | When `true` the test will be checking for ramp-down condition after each *scenario step*. When `false` the check will be done only at the end of a *scenario iteration*. | `false` |
+| `pace` | A dynamic pause after each *scenario iteration*. For example if the pace is 30s and one scenario iteration takes only 20s, the simulation will wait additional 10s before continuing to the next iteration. | `0` |
+| `userThinkTime` | Pause between individual scenario steps. | `5` |
+| `refreshTokenPeriod`| Period after which token should be refreshed. | `10` |
 
-`[-DuserThinkTime=SECONDS] [-DbadLoginAttempts=N] [-DrefreshTokenCount=N] [-DrefreshTokenPeriod=SECONDS]`
+#### Addtional Parameters of `keycloak.DefaultSimulation`
+
+| Parameter | Description | Default Value |
+| --- | --- | --- | 
+| `badLoginAttempts` | | `0`  |
+| `refreshTokenCount` | | `0` |
 
 
-If you want to run a different test you need to specify the test class name using `[-Dgatling.simulationClass=CLASSNAME]`.
+Example:
 
-For example:
-
-`mvn verify -Ptest -DrunUsers=1 -DnumOfIterations=10 -DuserThinkTime=0 -Ddataset=100u -DrefreshTokenPeriod=10 -Dgatling.simulationClass=keycloak.AdminSimulation`
+`mvn verify -Ptest -Dgatling.simulationClass=keycloak.AdminSimulation -Ddataset=100u -DrunUsers=1 -DsteadyLoadPeriod=30 -DuserThinkTime=0 -DrefreshTokenPeriod=15`
 
 
 ## Monitoring

testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java

@@ -7,6 +7,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Date;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadLocalRandom;
@@ -40,7 +41,6 @@ public class TestConfig {
     public static final String authPassword = System.getProperty("authPassword", "admin");
     public static final String authClient = System.getProperty("authClient", "admin-cli");
 
-
     //
     // Settings used by RealmsConfigurationBuilder to generate the dataset and by tests to work within constraints of the dataset
     //
@@ -52,24 +52,27 @@ public class TestConfig {
     public static final int clientRolesPerUser = Integer.getInteger("clientRolesPerUser", 2);
     public static final int clientRolesPerClient = Integer.getInteger("clientRolesPerClient", 2);
 
-
     //
     // Settings used by tests to control common test parameters
     //
     public static final int runUsers = Integer.getInteger("runUsers", 1);
     public static final int rampUpPeriod = Integer.getInteger("rampUpPeriod", 0);
+    public static final int steadyLoadPeriod = Integer.getInteger("steadyLoadPeriod", 30);
+    public static final boolean rampDownASAP = Boolean.getBoolean("rampDownASAP"); // check for rampdown condition after each scenario step
+    public static final int pace = Integer.getInteger("pace", 0); // additional dynamic "pause buffer" between scenario loops
     public static final int userThinkTime = Integer.getInteger("userThinkTime", 5);
     public static final int refreshTokenPeriod = Integer.getInteger("refreshTokenPeriod", 10);
 
+    // Computed timestamps
+    public static final long simulationStartTime = System.currentTimeMillis();//new Date().getTime();
+    public static final long rampDownPeriodStartTime = simulationStartTime + (rampUpPeriod + steadyLoadPeriod) * 1000;
 
     //
     // Settings used by DefaultSimulation to control behavior specific to DefaultSimulation
     //
-    public static final int numOfIterations = Integer.getInteger("numOfIterations", 1);
     public static final int badLoginAttempts = Integer.getInteger("badLoginAttempts", 0);
     public static final int refreshTokenCount = Integer.getInteger("refreshTokenCount", 0);
 
-
     public static final String serverUris;
     public static final List<String> serverUrisList;
 
@@ -97,7 +100,6 @@ public class TestConfig {
     // Clients iterators by realm
     private static final ConcurrentMap<String, Iterator<ClientInfo>> clientsIteratorMap = new ConcurrentHashMap<>();
 
-
     public static Iterator<UserInfo> getUsersIterator(String realm) {
         return usersIteratorMap.computeIfAbsent(realm, (k) -> randomUsersIterator(realm));
     }
@@ -111,11 +113,24 @@ public class TestConfig {
         return new FilteredIterator<>(clientsIt, (v) -> RealmsConfigurationBuilder.isClientConfidential(v.index));
     }
 
+    public static String toStringCommonTestParameters() {
+        return String.format(
+        "  runUsers: %s\n" + 
+        "  rampUpPeriod: %s\n"+ 
+        "  steadyLoadPeriod: %s\n"+
+        "  rampDownASAP: %s\n"+ 
+        "  pace: %s\n"+ 
+        "  userThinkTime: %s\n"+ 
+        "  refreshTokenPeriod: %s",
+                runUsers, rampUpPeriod, steadyLoadPeriod, rampDownASAP, pace, userThinkTime, refreshTokenPeriod
+        );
+    }
+
     public static String toStringDatasetProperties() {
         return String.format("  numOfRealms: %s\n  usersPerRealm: %s\n  clientsPerRealm: %s\n  realmRoles: %s\n  realmRolesPerUser: %s\n  clientRolesPerUser: %s\n  clientRolesPerClient: %s\n  hashIterations: %s",
                 numOfRealms, usersPerRealm, clientsPerRealm, realmRoles, realmRolesPerUser, clientRolesPerUser, clientRolesPerClient, hashIterations);
     }
-
+    
     public static Iterator<UserInfo> sequentialUsersIterator(final String realm) {
 
         return new Iterator<UserInfo>() {
@@ -200,4 +215,5 @@ public class TestConfig {
             throw new RuntimeException("Can't have more clientRolesPerUser than there are all client roles (clientsPerRealm * clientRolesPerClient)");
         }
     }
+    
 }

testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala

@@ -1,6 +1,7 @@
 package keycloak
 
 import io.gatling.core.Predef._
+import io.gatling.core.validation.Validation
 import io.gatling.http.Predef._
 import org.jboss.perf.util.Util
 import org.keycloak.performance.TestConfig
@@ -16,14 +17,7 @@ class AdminConsoleSimulation extends Simulation {
   println()
   println("Target server: " + TestConfig.serverUrisList.get(0))
   println()
-  println("Using test parameters:")
-  println("  runUsers: " + TestConfig.runUsers)
-  println("  numOfIterations: " + TestConfig.numOfIterations)
-  println("  rampUpPeriod: " + TestConfig.rampUpPeriod)
-  println("  userThinkTime: " + TestConfig.userThinkTime)
-  //println("  badLoginAttempts: " + TestConfig.badLoginAttempts)
-  //println("  refreshTokenCount: " + TestConfig.refreshTokenCount)
-  //println("  refreshTokenPeriod: " + TestConfig.refreshTokenPeriod)
+  println("Using test parameters:\n" + TestConfig.toStringCommonTestParameters);
   println()
   println("Using dataset properties:\n" + TestConfig.toStringDatasetProperties)
 
@@ -110,16 +104,18 @@ class AdminConsoleSimulation extends Simulation {
 
 
   val adminScenario = scenario("AdminConsole")
-    .repeat(TestConfig.numOfIterations) {
+    .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) {
+      pace(TestConfig.pace)
       adminSession
     }
 
+  setUp(adminScenario
+    .inject(rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod)
+    .protocols(httpProtocol))
 
-  setUp(adminScenario.inject({
-    if (TestConfig.rampUpPeriod > 0) {
-      rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod
-    } else {
-      atOnceUsers(TestConfig.runUsers)
-    }
-  }).protocols(httpProtocol))
-}
+  
+  def rampDownPeriodNotReached(): Validation[Boolean] = {
+    System.currentTimeMillis < TestConfig.rampDownPeriodStartTime
+  }
+  
+}

testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala

@@ -26,16 +26,12 @@ class DefaultSimulation extends Simulation {
 
 
   println()
-  println("Taget servers: " + TestConfig.serverUrisList)
+  println("Target servers: " + TestConfig.serverUrisList)
   println()
-  println("Using test parameters:")
-  println("  runUsers: " + TestConfig.runUsers)
-  println("  numOfIterations: " + TestConfig.numOfIterations)
-  println("  rampUpPeriod: " + TestConfig.rampUpPeriod)
-  println("  userThinkTime: " + TestConfig.userThinkTime)
-  println("  badLoginAttempts: " + TestConfig.badLoginAttempts)
+
+  println("Using test parameters:\n" + TestConfig.toStringCommonTestParameters);
   println("  refreshTokenCount: " + TestConfig.refreshTokenCount)
-  println("  refreshTokenPeriod: " + TestConfig.refreshTokenPeriod)
+  println("  badLoginAttempts: " + TestConfig.badLoginAttempts)
   println()
   println("Using dataset properties:\n" + TestConfig.toStringDatasetProperties)
 
@@ -137,20 +133,14 @@ class DefaultSimulation extends Simulation {
       .check(status.is(302), header("Location").is("${appUrl}")))
 
   val usersScenario = scenario("users")
-    .repeat(TestConfig.numOfIterations) {
+    .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) {
+      pace(TestConfig.pace)
       userSession
     }
 
-  setUp(usersScenario.inject( {
-    if (TestConfig.rampUpPeriod > 0) {
-      rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod
-    } else {
-      atOnceUsers(TestConfig.runUsers)
-    }
-  }).protocols(httpDefault))
-
-
-
+  setUp(usersScenario
+    .inject(rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod)
+    .protocols(httpDefault))
 
   //
   // Function definitions
@@ -163,4 +153,9 @@ class DefaultSimulation extends Simulation {
     }
     missCounter.getAndDecrement() > 0
   }
+
+  def rampDownPeriodNotReached(): Validation[Boolean] = {
+    System.currentTimeMillis < TestConfig.rampDownPeriodStartTime
+  }
+
 }