From 5fd00a0def44822e8a466c5e18552177b8eed8b4 Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Thu, 9 Nov 2017 23:57:00 +0100 Subject: [PATCH] KEYCLOAK-5971 Replace "numOfIterations" parameter with "steadyLoadPeriod" --- testsuite/performance/README.md | 53 ++++++++++++++----- .../org/keycloak/performance/TestConfig.java | 28 +++++++--- .../keycloak/AdminConsoleSimulation.scala | 30 +++++------ .../scala/keycloak/DefaultSimulation.scala | 33 +++++------- 4 files changed, 89 insertions(+), 55 deletions(-) diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index e229824395..cccedd54ef 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -98,41 +98,68 @@ Dataset properties are loaded from `datasets/${dataset}.properties` file. Indivi Dataset data is first generated as a .json file, and then imported into Keycloak via Admin Client REST API. +#### Dataset Properties + +| Property | Description | Value in the Default Dataset | +| --- | --- | --- | +| `numOfRealms` | Number of realms to be created. | `1` | +| `usersPerRealm` | Number of users per realm. | `100` | +| `clientsPerRealm` | Number of clients per realm. | `2` | +| `realmRoles` | Number of realm-roles per realm. | `2` | +| `realmRolesPerUser` | Number of realm-roles assigned to a created user. Has to be less than or equal to `realmRoles`. | `2` | +| `clientRolesPerUser` | Number of client-roles assigned to a created user. Has to be less than or equal to `clientsPerRealm * clientRolesPerClient`. | `2` | +| `clientRolesPerClient` | Number of client-roles per created client. | `2` | +| `hashIterations` | Number of password hashing iterations. | `27500` | + + #### Examples: - `mvn verify -Pgenerate-data` - generate default dataset - `mvn verify -Pgenerate-data -DusersPerRealm=5` - generate default dataset, override the `usersPerRealm` property - `mvn verify -Pgenerate-data -Ddataset=100u` - generate `100u` dataset - `mvn verify -Pgenerate-data -Ddataset=100r/default` - generate dataset based on `datasets/100r/default.properties` -The data can also be exported from the database, and stored locally as `datasets/${dataset}.sql.gz` -`DATASET=100u ./prepare-dump.sh` +#### Export / Import Database Dump To speed up dataset initialization part, it is possible to pass `-Dexport-dump` option to have the generated dataset exported right after it has been generated. Then, if there is a data dump file available then `-Pimport-dump` can be used to import the data directly into the database, bypassing Keycloak server completely. -Usage: `mvn verify -Pimport-dump [-Ddataset=DATASET]` +**Usage:** `mvn verify -Pimport-dump [-Ddataset=DATASET]` -#### Example: -- `mvn verify -Pimport-dump -Ddataset=100u` - import `datasets/100u.sql.gz` dump file created using `prepare-dump.sh`. +**For example:** +- `mvn verify -Pgenerate-data -Ddataset=100u -Dexport-dump` will generate data based on `datasets/100u.properties` and export a database dump to a file: `datasets/100u.sql.gz`. +- `mvn verify -Pimport-dump -Ddataset=100u` will import the database dump from a file: `datasets/100u.sql.gz`, and reboot the server(s) ### Run Tests -Usage: `mvn verify -Ptest [-DrunUsers=N] [-DrampUpPeriod=SECONDS] [-DnumOfIterations=N] [-Ddataset=DATASET] [-D=]* [-D=]* `. +Usage: `mvn verify -Ptest[,cluster] [-DtestParameter=value]`. -_*Note:* The same dataset properties which were used for data generation/import should be supplied to the `test` phase._ +#### Common Parameters -The default test `keycloak.DefaultSimulation` takes the following additional properties: +| Parameter | Description | Default Value | +| --- | --- | --- | +| `gatling.simulationClass` | Classname of the simulation to be run. | `keycloak.DefaultSimulation` | +| `dataset` | Name of the dataset to use. (Individual dataset properties can be overridden with `-Ddataset.property=value`.) | `default` | +| `runUsers` | Number of users for the simulation run. | `1` | +| `rampUpPeriod` | Period during which the users will be ramped up. (seconds) | `0` | +| `steadyLoadPeriod` | A period of steady load. (seconds) | `30` | +| `rampDownASAP` | When `true` the test will be checking for ramp-down condition after each *scenario step*. When `false` the check will be done only at the end of a *scenario iteration*. | `false` | +| `pace` | A dynamic pause after each *scenario iteration*. For example if the pace is 30s and one scenario iteration takes only 20s, the simulation will wait additional 10s before continuing to the next iteration. | `0` | +| `userThinkTime` | Pause between individual scenario steps. | `5` | +| `refreshTokenPeriod`| Period after which token should be refreshed. | `10` | -`[-DuserThinkTime=SECONDS] [-DbadLoginAttempts=N] [-DrefreshTokenCount=N] [-DrefreshTokenPeriod=SECONDS]` +#### Addtional Parameters of `keycloak.DefaultSimulation` + +| Parameter | Description | Default Value | +| --- | --- | --- | +| `badLoginAttempts` | | `0` | +| `refreshTokenCount` | | `0` | -If you want to run a different test you need to specify the test class name using `[-Dgatling.simulationClass=CLASSNAME]`. +Example: -For example: - -`mvn verify -Ptest -DrunUsers=1 -DnumOfIterations=10 -DuserThinkTime=0 -Ddataset=100u -DrefreshTokenPeriod=10 -Dgatling.simulationClass=keycloak.AdminSimulation` +`mvn verify -Ptest -Dgatling.simulationClass=keycloak.AdminSimulation -Ddataset=100u -DrunUsers=1 -DsteadyLoadPeriod=30 -DuserThinkTime=0 -DrefreshTokenPeriod=15` ## Monitoring diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java index e95ca8782a..4fb33b71f2 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Date; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadLocalRandom; @@ -40,7 +41,6 @@ public class TestConfig { public static final String authPassword = System.getProperty("authPassword", "admin"); public static final String authClient = System.getProperty("authClient", "admin-cli"); - // // Settings used by RealmsConfigurationBuilder to generate the dataset and by tests to work within constraints of the dataset // @@ -52,24 +52,27 @@ public class TestConfig { public static final int clientRolesPerUser = Integer.getInteger("clientRolesPerUser", 2); public static final int clientRolesPerClient = Integer.getInteger("clientRolesPerClient", 2); - // // Settings used by tests to control common test parameters // public static final int runUsers = Integer.getInteger("runUsers", 1); public static final int rampUpPeriod = Integer.getInteger("rampUpPeriod", 0); + public static final int steadyLoadPeriod = Integer.getInteger("steadyLoadPeriod", 30); + public static final boolean rampDownASAP = Boolean.getBoolean("rampDownASAP"); // check for rampdown condition after each scenario step + public static final int pace = Integer.getInteger("pace", 0); // additional dynamic "pause buffer" between scenario loops public static final int userThinkTime = Integer.getInteger("userThinkTime", 5); public static final int refreshTokenPeriod = Integer.getInteger("refreshTokenPeriod", 10); + // Computed timestamps + public static final long simulationStartTime = System.currentTimeMillis();//new Date().getTime(); + public static final long rampDownPeriodStartTime = simulationStartTime + (rampUpPeriod + steadyLoadPeriod) * 1000; // // Settings used by DefaultSimulation to control behavior specific to DefaultSimulation // - public static final int numOfIterations = Integer.getInteger("numOfIterations", 1); public static final int badLoginAttempts = Integer.getInteger("badLoginAttempts", 0); public static final int refreshTokenCount = Integer.getInteger("refreshTokenCount", 0); - public static final String serverUris; public static final List serverUrisList; @@ -97,7 +100,6 @@ public class TestConfig { // Clients iterators by realm private static final ConcurrentMap> clientsIteratorMap = new ConcurrentHashMap<>(); - public static Iterator getUsersIterator(String realm) { return usersIteratorMap.computeIfAbsent(realm, (k) -> randomUsersIterator(realm)); } @@ -111,11 +113,24 @@ public class TestConfig { return new FilteredIterator<>(clientsIt, (v) -> RealmsConfigurationBuilder.isClientConfidential(v.index)); } + public static String toStringCommonTestParameters() { + return String.format( + " runUsers: %s\n" + + " rampUpPeriod: %s\n"+ + " steadyLoadPeriod: %s\n"+ + " rampDownASAP: %s\n"+ + " pace: %s\n"+ + " userThinkTime: %s\n"+ + " refreshTokenPeriod: %s", + runUsers, rampUpPeriod, steadyLoadPeriod, rampDownASAP, pace, userThinkTime, refreshTokenPeriod + ); + } + public static String toStringDatasetProperties() { return String.format(" numOfRealms: %s\n usersPerRealm: %s\n clientsPerRealm: %s\n realmRoles: %s\n realmRolesPerUser: %s\n clientRolesPerUser: %s\n clientRolesPerClient: %s\n hashIterations: %s", numOfRealms, usersPerRealm, clientsPerRealm, realmRoles, realmRolesPerUser, clientRolesPerUser, clientRolesPerClient, hashIterations); } - + public static Iterator sequentialUsersIterator(final String realm) { return new Iterator() { @@ -200,4 +215,5 @@ public class TestConfig { throw new RuntimeException("Can't have more clientRolesPerUser than there are all client roles (clientsPerRealm * clientRolesPerClient)"); } } + } diff --git a/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala index 97e323fedf..d588582e58 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/AdminConsoleSimulation.scala @@ -1,6 +1,7 @@ package keycloak import io.gatling.core.Predef._ +import io.gatling.core.validation.Validation import io.gatling.http.Predef._ import org.jboss.perf.util.Util import org.keycloak.performance.TestConfig @@ -16,14 +17,7 @@ class AdminConsoleSimulation extends Simulation { println() println("Target server: " + TestConfig.serverUrisList.get(0)) println() - println("Using test parameters:") - println(" runUsers: " + TestConfig.runUsers) - println(" numOfIterations: " + TestConfig.numOfIterations) - println(" rampUpPeriod: " + TestConfig.rampUpPeriod) - println(" userThinkTime: " + TestConfig.userThinkTime) - //println(" badLoginAttempts: " + TestConfig.badLoginAttempts) - //println(" refreshTokenCount: " + TestConfig.refreshTokenCount) - //println(" refreshTokenPeriod: " + TestConfig.refreshTokenPeriod) + println("Using test parameters:\n" + TestConfig.toStringCommonTestParameters); println() println("Using dataset properties:\n" + TestConfig.toStringDatasetProperties) @@ -110,16 +104,18 @@ class AdminConsoleSimulation extends Simulation { val adminScenario = scenario("AdminConsole") - .repeat(TestConfig.numOfIterations) { + .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) { + pace(TestConfig.pace) adminSession } + setUp(adminScenario + .inject(rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod) + .protocols(httpProtocol)) - setUp(adminScenario.inject({ - if (TestConfig.rampUpPeriod > 0) { - rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod - } else { - atOnceUsers(TestConfig.runUsers) - } - }).protocols(httpProtocol)) -} \ No newline at end of file + + def rampDownPeriodNotReached(): Validation[Boolean] = { + System.currentTimeMillis < TestConfig.rampDownPeriodStartTime + } + +} diff --git a/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala index a2ed6d4830..3f0d6e7ba0 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/DefaultSimulation.scala @@ -26,16 +26,12 @@ class DefaultSimulation extends Simulation { println() - println("Taget servers: " + TestConfig.serverUrisList) + println("Target servers: " + TestConfig.serverUrisList) println() - println("Using test parameters:") - println(" runUsers: " + TestConfig.runUsers) - println(" numOfIterations: " + TestConfig.numOfIterations) - println(" rampUpPeriod: " + TestConfig.rampUpPeriod) - println(" userThinkTime: " + TestConfig.userThinkTime) - println(" badLoginAttempts: " + TestConfig.badLoginAttempts) + + println("Using test parameters:\n" + TestConfig.toStringCommonTestParameters); println(" refreshTokenCount: " + TestConfig.refreshTokenCount) - println(" refreshTokenPeriod: " + TestConfig.refreshTokenPeriod) + println(" badLoginAttempts: " + TestConfig.badLoginAttempts) println() println("Using dataset properties:\n" + TestConfig.toStringDatasetProperties) @@ -137,20 +133,14 @@ class DefaultSimulation extends Simulation { .check(status.is(302), header("Location").is("${appUrl}"))) val usersScenario = scenario("users") - .repeat(TestConfig.numOfIterations) { + .asLongAs(s => rampDownPeriodNotReached(), null, TestConfig.rampDownASAP) { + pace(TestConfig.pace) userSession } - setUp(usersScenario.inject( { - if (TestConfig.rampUpPeriod > 0) { - rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod - } else { - atOnceUsers(TestConfig.runUsers) - } - }).protocols(httpDefault)) - - - + setUp(usersScenario + .inject(rampUsers(TestConfig.runUsers) over TestConfig.rampUpPeriod) + .protocols(httpDefault)) // // Function definitions @@ -163,4 +153,9 @@ class DefaultSimulation extends Simulation { } missCounter.getAndDecrement() > 0 } + + def rampDownPeriodNotReached(): Validation[Boolean] = { + System.currentTimeMillis < TestConfig.rampDownPeriodStartTime + } + }