From dabf4e8bbd96e20acd6b6bb7f566cc5333be5328 Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Mon, 19 Feb 2018 11:13:30 +0100 Subject: [PATCH] KEYCLOAK-6165 Ability to load performance provisioning and test parameters from a properties file --- testsuite/performance/README.md | 150 ++++++------ testsuite/performance/keycloak/pom.xml | 2 +- testsuite/performance/pom.xml | 1 + testsuite/performance/tests/docker-compose.sh | 9 +- .../datasets/100r/100u2c.properties} | 0 .../datasets/100r/2u2c.properties} | 0 .../datasets/100u2c.properties} | 0 .../datasets/200ku200c.properties | 0 .../datasets/200ku2kc.properties | 0 .../datasets/2ku200c.properties | 0 .../datasets/2u2c.properties} | 0 .../datasets/500ku500c.properties | 0 .../datasets/volume-testing/big.properties | 8 + .../datasets/volume-testing/medium.properties | 8 + .../datasets/volume-testing/small.properties | 8 + .../docker-compose/4cpus/cluster.properties | 32 +++ .../docker-compose/4cpus/crossdc.properties | 42 ++++ .../4cpus/singlenode.properties | 23 ++ .../parameters/test/admin-console.properties | 8 + .../parameters/test/basic-oidc.properties | 10 + testsuite/performance/tests/pom.xml | 215 +++++------------- .../tests/prepare-provisioning.xml | 24 ++ .../org/keycloak/performance/TestConfig.java | 2 - 23 files changed, 316 insertions(+), 226 deletions(-) rename testsuite/performance/tests/{datasets/100r/100u.properties => parameters/datasets/100r/100u2c.properties} (100%) rename testsuite/performance/tests/{datasets/100r/default.properties => parameters/datasets/100r/2u2c.properties} (100%) rename testsuite/performance/tests/{datasets/100u.properties => parameters/datasets/100u2c.properties} (100%) rename testsuite/performance/tests/{ => parameters}/datasets/200ku200c.properties (100%) rename testsuite/performance/tests/{ => parameters}/datasets/200ku2kc.properties (100%) rename testsuite/performance/tests/{ => parameters}/datasets/2ku200c.properties (100%) rename testsuite/performance/tests/{datasets/default.properties => parameters/datasets/2u2c.properties} (100%) rename testsuite/performance/tests/{ => parameters}/datasets/500ku500c.properties (100%) create mode 100644 testsuite/performance/tests/parameters/datasets/volume-testing/big.properties create mode 100644 testsuite/performance/tests/parameters/datasets/volume-testing/medium.properties create mode 100644 testsuite/performance/tests/parameters/datasets/volume-testing/small.properties create mode 100644 testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/cluster.properties create mode 100644 testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/crossdc.properties create mode 100644 testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/singlenode.properties create mode 100644 testsuite/performance/tests/parameters/test/admin-console.properties create mode 100644 testsuite/performance/tests/parameters/test/basic-oidc.properties create mode 100644 testsuite/performance/tests/prepare-provisioning.xml diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index 24b12ba1a8..e770eca273 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -26,9 +26,8 @@ mvn clean install # Make sure your Docker daemon is running THEN mvn verify -Pprovision -mvn verify -Pgenerate-data -Ddataset=100u -DnumOfWorkers=10 -DhashIterations=100 -mvn verify -Ptest -Ddataset=100u -DusersPerSec=4.5 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DmeasurementPeriod=60 -DfilterResults=true - +mvn verify -Pgenerate-data -Ddataset=100u2c -DnumOfWorkers=10 -DhashIterations=100 +mvn verify -Ptest -Ddataset=100u2c -DusersPerSec=2 -DrampUpPeriod=10 -DuserThinkTime=0 -DbadLoginAttempts=1 -DrefreshTokenCount=1 -DmeasurementPeriod=60 -DfilterResults=true ``` Now open the generated report in a browser - the link to .html file is displayed at the end of the test. @@ -40,7 +39,7 @@ mvn verify -Pteardown You can perform all phases in a single run: ``` -mvn verify -Pprovision,generate-data,test,teardown -Ddataset=100u -DnumOfWorkers=10 -DhashIterations=100 -DusersPerSec=5 -DrampUpPeriod=10 +mvn verify -Pprovision,generate-data,test,teardown -Ddataset=100u2c -DnumOfWorkers=10 -DhashIterations=100 -DusersPerSec=4 -DrampUpPeriod=10 ``` Note: The order in which maven profiles are listed does not determine the order in which profile related plugins are executed. `teardown` profile always executes last. @@ -49,21 +48,44 @@ Keep reading for more information. ## Provisioning -### Available provisioners: - -- `docker-compose` **Default.** See [`README.docker-compose.md`](README.docker-compose.md) for more details. - ### Provision -Usage: `mvn verify -Pprovision [-Dprovisioner=] [-D=] …`. +#### Provisioners + +Depending on the target environment different provisioners may be used. +Provisioner can be selected via property `-Dprovisioner=PROVISIONER`. + +Default value is `docker-compose` which is intended for testing on a local docker host. +This is currently the only implemented option. See [`README.docker-compose.md`](README.docker-compose.md) for more details. #### Deployment Types -- Single node: `mvn verify -Pprovision` -- Cluster: `mvn verify -Pprovision,cluster [-Dkeycloak.scale=N] [-Dkeycloak.cpusets="cpuset1 cpuset2 … cpusetM"]`. `N ∈ {1 .. M}`. -- Cross-DC: `mvn verify -Pprovision,crossdc [-Dkeycloak.dc1.scale=K] [-Dkeycloak.dc2.scale=L] [-Dkeycloak.dc1.cpusets=…] [-Dkeycloak.dc2.cpusets=…]` +Different types of deployment can be provisioned. +The default deployment is `singlenode` with only a single instance of Keycloak server and a database. +Additional options are `cluster` and `crossdc` which can be enabled with a profile (see below). + +#### Usage + +Usage: `mvn verify -P provision[,DEPLOYMENT_PROFILE] [-Dprovisioning.properties=NAMED_PROPERTY_SET]`. + +The properties are loaded from `tests/parameters/provisioning/${provisioning.properties}.properties` file. +Individual parameters can be overriden from command line via `-D` params. + +Default property set is `docker-compose/4cpus/singlenode`. + +To load a custom properties file specify `-Dprovisioning.properties.file=ABSOLUTE_PATH_TO_FILE` instead of `-Dprovisioning.properties`. +This file needs to contain all properties required by the specific combination of provisioner and deployment type. +See examples in folder `tests/parameters/provisioning/docker-compose/4cpus`. + +Available parameters are described in [`README.provisioning-parameters.md`](README.provisioning-parameters.md). + +#### Examples: +- Provision a single-node deployment with docker-compose: `mvn verify -P provision` +- Provision a cluster deployment with docker-compose: `mvn verify -P provision,cluster` +- Provision a cluster deployment with docker-compose, overriding some properties: `mvn verify -P provision,cluster -Dkeycloak.scale=2 -Dlb.worker.task-max-threads=32` +- Provision a cross-DC deployment with docker-compose: `mvn verify -P provision,crossdc` +- Provision a cross-DC deployment with docker-compose using a custom properties file: `mvn verify -P provision,crossdc -Dprovisioning.properties.file=/tmp/custom-crossdc.properties` -All available parameters are described in [`README.provisioning-parameters.md`](README.provisioning-parameters.md). #### Provisioned System @@ -71,8 +93,9 @@ The `provision` operation will produce a `provisioned-system.properties` inside with information about the provisioned system such as the type of deployment and URLs of Keycloak servers and load balancers. This information is then used by operations `generate-data`, `import-dump`, `test`, `teardown`. -Provisioning can be run multiple times with different parameters. The system will be updated/reprovisioned based on the new parameters. -However when switching between different deployment types (e.g. from `singlenode` to `cluster`) it is always necessary +Provisioning operation is idempotent for a specific combination of provisioner+deployment. +When running multiple times the system will be simply updated based on the new parameters. +However when switching between different provisioiners or deployment types it is **always necessary** to tear down the currently running system. **Note:** When switching deployment type from `singlenode` or `cluster` to `crossdc` (or the other way around) @@ -92,18 +115,19 @@ because it contains the `provisioned-system.properties` with information about t ### Generate Test Data -Usage: `mvn verify -Pgenerate-data [-Ddataset=DATASET] [-D=]`. +Usage: `mvn verify -P generate-data [-Ddataset=NAMED_PROPERTY_SET] [-DnumOfWorkers=N]`. The default dataset is `2u2c`. Workers default to `1`. -Dataset properties are loaded from `datasets/${dataset}.properties` file. Individual properties can be overriden by specifying `-D` params. +The parameters are loaded from `tests/parameters/datasets/${dataset}.properties` file. +Individual properties can be overriden from command line via `-D` params. -Dataset data is first generated as a .json file, and then imported into Keycloak via Admin Client REST API. +To use a custom properties file specify `-Ddataset.properties.file=ABSOLUTE_PATH_TO_FILE` instead of `-Ddataset`. -#### Dataset Properties +#### Dataset Parameters | Property | Description | Value in the Default Dataset | | --- | --- | --- | | `numOfRealms` | Number of realms to be created. | `1` | -| `usersPerRealm` | Number of users per realm. | `100` | +| `usersPerRealm` | Number of users per realm. | `2` | | `clientsPerRealm` | Number of clients per realm. | `2` | | `realmRoles` | Number of realm-roles per realm. | `2` | | `realmRolesPerUser` | Number of realm-roles assigned to a created user. Has to be less than or equal to `realmRoles`. | `2` | @@ -113,53 +137,73 @@ Dataset data is first generated as a .json file, and then imported into Keycloak #### Examples: -- `mvn verify -Pgenerate-data` - generate default dataset -- `mvn verify -Pgenerate-data -DusersPerRealm=5` - generate default dataset, override the `usersPerRealm` property -- `mvn verify -Pgenerate-data -Ddataset=100u` - generate `100u` dataset -- `mvn verify -Pgenerate-data -Ddataset=100r/default` - generate dataset based on `datasets/100r/default.properties` +- Generate the default dataset. `mvn verify -P generate-data` +- Generate the `100u2c` dataset. `mvn verify -P generate-data -Ddataset=100u2c` +- Generate the `100u2c` dataset but override some parameters. `mvn verify -P generate-data -Ddataset=100u2c -DclientRolesPerUser=5 -DclientRolesPerClient=5` -#### Export / Import Database Dump +#### Export Database -To speed up dataset initialization part, it is possible to pass `-Dexport-dump` option to have the generated dataset -exported right after it has been generated. Then, if there is a data dump file available then `-Pimport-dump` -can be used to import the data directly into the database, bypassing Keycloak server completely. +To export the generated data to a data-dump file enable profile `-P export-dump`. This will create a `${DATASET}.sql.gz` file next to the dataset properties file. -**Usage:** `mvn verify -Pimport-dump [-Ddataset=DATASET]` +Example: `mvn verify -P generate-data,export-dump -Ddataset=100u2c` -**For example:** -- `mvn verify -Pgenerate-data -Ddataset=100u -Dexport-dump` will generate data based on `datasets/100u.properties` and export a database dump to a file: `datasets/100u.sql.gz`. -- `mvn verify -Pimport-dump -Ddataset=100u` will import the database dump from a file: `datasets/100u.sql.gz`, and reboot the server(s) +#### Import Database + +To import data from an existing data-dump file use profile `-P import-dump`. + +Example: `mvn verify -P import-dump -Ddataset=100u2c` + +If the dump file doesn't exist locally the script will attempt to download it from `${db.dump.download.site}` which defaults to `https://downloads.jboss.org/keycloak-qe/${server.version}` +with `server.version` defaulting to `${project.version}` from `pom.xml`. + +**Warning:** Don't override dataset parameters (with `-Dparam=value`) when running export/import because then the contents of dump file might not match the properties file. ### Run Tests -Usage: `mvn verify -Ptest[,cluster] [-DtestParameter=value]`. +Usage: `mvn verify -P test [-Dtest.properties=NAMED_PROPERTY_SET]`. Default property set is `basic-oidc`. -#### Common Parameters +The parameters are loaded from `tests/parameters/test/${test.properties}.properties` file. +Individual properties can be overriden from command line via `-D` params. + +To use a custom properties file specify `-Dtest.properties.file=ABSOLUTE_PATH_TO_FILE` instead of `-Dtest.properties`. + +When running the tests it is also necessary to define a dataset to use. Usage is described in the section above. + +#### Common Test Run Parameters | Parameter | Description | Default Value | | --- | --- | --- | | `gatling.simulationClass` | Classname of the simulation to be run. | `keycloak.BasicOIDCSimulation` | | `dataset` | Name of the dataset to use. (Individual dataset properties can be overridden with `-Ddataset.property=value`.) | `default` | -| `usersPerSec` | Arrival rate of new users per second. Can be a floating point number. | `1.0` | -| `rampUpPeriod` | Period during which the users will be ramped up. (seconds) | `0` | -| `warmUpPeriod` | Period with steady number of users intended for the system under test to warm up. (seconds) | `0` | +| `usersPerSec` | Arrival rate of new users per second. Can be a floating point number. | `1.0` for BasicOIDCSimulation, `0.2` for AdminConsoleSimulation | +| `rampUpPeriod` | Period during which the users will be ramped up. (seconds) | `15` | +| `warmUpPeriod` | Period with steady number of users intended for the system under test to warm up. (seconds) | `15` | | `measurementPeriod` | A measurement period after the system is warmed up. (seconds) | `30` | | `filterResults` | Whether to filter out requests which are outside of the `measurementPeriod`. | `false` | | `userThinkTime` | Pause between individual scenario steps. | `5` | | `refreshTokenPeriod`| Period after which token should be refreshed. | `10` | -#### Addtional Parameters of `keycloak.BasicOIDCSimulation` +#### Test Run Parameters specific to `BasicOIDCSimulation` | Parameter | Description | Default Value | | --- | --- | --- | | `badLoginAttempts` | | `0` | | `refreshTokenCount` | | `0` | +#### Examples: -Example: +- Run test with default test and dataset parameters: -`mvn verify -Ptest -Dgatling.simulationClass=keycloak.AdminConsoleSimulation -Ddataset=100u -DusersPerSec=1 -DmeasurementPeriod=60 -DuserThinkTime=0 -DrefreshTokenPeriod=15` +`mvn verify -P test` + +- Run test specific test and dataset parameters: + +`mvn verify -P test -Dtest.properties=basic-oidc -Ddataset=100u2c` + +- Run test with specific test and dataset parameters, overriding some from command line: + +`mvn verify -P test -Dtest.properties=admin-console -Ddataset=100u2c -DrampUpPeriod=30 -DwarmUpPeriod=60 -DusersPerSec=0.3` ## Monitoring @@ -202,32 +246,6 @@ To compress the binary output with bzip add `-Dbzip=true` to the commandline. Results will be stored in folder: `tests/target/sar`. -## Examples - -### Single-node - -- Provision single node of KC + DB, generate data, run test, and tear down the provisioned system: - - `mvn verify -Pprovision,generate-data,test,teardown -Ddataset=100u -DusersPerSec=5` - -- Provision single node of KC + DB, generate data, no test, no teardown: - - `mvn verify -Pprovision,generate-data -Ddataset=100u` - -- Run test against provisioned system generating 5 new users per second, ramped up over 10 seconds, then tear it down: - - `mvn verify -Ptest,teardown -Ddataset=100u -DusersPerSec=5 -DrampUpPeriod=10` - -### Cluster - -- Provision a 1-node KC cluster + DB, generate data, run test against the provisioned system, then tear it down: - - `mvn verify -Pprovision,cluster,generate-data,test,teardown -Ddataset=100u -DusersPerSec=5` - -- Provision a 2-node KC cluster + DB, generate data, run test against the provisioned system, then tear it down: - - `mvn verify -Pprovision,cluster,generate-data,test,teardown -Dkeycloak.scale=2 -DusersPerRealm=200 -DusersPerSec=5` - ## Developing tests in IntelliJ IDEA diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml index f9404d9dd3..2e1d1b5df2 100644 --- a/testsuite/performance/keycloak/pom.xml +++ b/testsuite/performance/keycloak/pom.xml @@ -37,7 +37,7 @@ org.keycloak keycloak-server-dist - ${product.version} + keycloak-${server.version} ${project.build.directory}/${server.unpacked.folder.name} diff --git a/testsuite/performance/pom.xml b/testsuite/performance/pom.xml index 089fcd0a54..5865c061d6 100644 --- a/testsuite/performance/pom.xml +++ b/testsuite/performance/pom.xml @@ -32,6 +32,7 @@ pom + ${product.version} diff --git a/testsuite/performance/tests/docker-compose.sh b/testsuite/performance/tests/docker-compose.sh index 68529ebc9e..045dbfd5bf 100755 --- a/testsuite/performance/tests/docker-compose.sh +++ b/testsuite/performance/tests/docker-compose.sh @@ -366,13 +366,16 @@ case "$OPERATION" in crossdc) export DB_CONTAINER=${PROJECT_NAME}_mariadb_dc1_1 ;; *) echo "Deployment '$DEPLOYMENT' doesn't support operation '$OPERATION'." ; exit 1 ;; esac - if [ -z "$DATASET" ]; then echo "Operation '$OPERATION' requires DATASET parameter."; exit 1; fi + if [ ! -f "$DATASET_PROPERTIES_FILE" ]; then echo "Operation '$OPERATION' requires a valid DATASET_PROPERTIES_FILE parameter."; exit 1; fi + DATASET_PROPERTIES_FILENAME=`basename $DATASET_PROPERTIES_FILE` + DATASET=${DATASET_PROPERTIES_FILENAME%.properties} + echo "DATASET_PROPERTIES_FILE: $DATASET_PROPERTIES_FILE" echo "DATASET: $DATASET" echo "Stopping Keycloak services." runCommand "docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} stop $KEYCLOAK_SERVICES" - cd $PROJECT_BASEDIR/datasets + cd `dirname $DATASET_PROPERTIES_FILE` case "$OPERATION" in export-dump) echo "Exporting $DATASET.sql." @@ -384,7 +387,7 @@ case "$OPERATION" in import-dump) DUMP_DOWNLOAD_SITE=${DUMP_DOWNLOAD_SITE:-https://downloads.jboss.org/keycloak-qe} if [ ! -f "$DATASET.sql.gz" ]; then - echo "Downloading dump file." + echo "Downloading dump file: $DUMP_DOWNLOAD_SITE/$DATASET.sql.gz" if ! curl -f -O $DUMP_DOWNLOAD_SITE/$DATASET.properties -O $DUMP_DOWNLOAD_SITE/$DATASET.sql.gz ; then echo Download failed. exit 1 diff --git a/testsuite/performance/tests/datasets/100r/100u.properties b/testsuite/performance/tests/parameters/datasets/100r/100u2c.properties similarity index 100% rename from testsuite/performance/tests/datasets/100r/100u.properties rename to testsuite/performance/tests/parameters/datasets/100r/100u2c.properties diff --git a/testsuite/performance/tests/datasets/100r/default.properties b/testsuite/performance/tests/parameters/datasets/100r/2u2c.properties similarity index 100% rename from testsuite/performance/tests/datasets/100r/default.properties rename to testsuite/performance/tests/parameters/datasets/100r/2u2c.properties diff --git a/testsuite/performance/tests/datasets/100u.properties b/testsuite/performance/tests/parameters/datasets/100u2c.properties similarity index 100% rename from testsuite/performance/tests/datasets/100u.properties rename to testsuite/performance/tests/parameters/datasets/100u2c.properties diff --git a/testsuite/performance/tests/datasets/200ku200c.properties b/testsuite/performance/tests/parameters/datasets/200ku200c.properties similarity index 100% rename from testsuite/performance/tests/datasets/200ku200c.properties rename to testsuite/performance/tests/parameters/datasets/200ku200c.properties diff --git a/testsuite/performance/tests/datasets/200ku2kc.properties b/testsuite/performance/tests/parameters/datasets/200ku2kc.properties similarity index 100% rename from testsuite/performance/tests/datasets/200ku2kc.properties rename to testsuite/performance/tests/parameters/datasets/200ku2kc.properties diff --git a/testsuite/performance/tests/datasets/2ku200c.properties b/testsuite/performance/tests/parameters/datasets/2ku200c.properties similarity index 100% rename from testsuite/performance/tests/datasets/2ku200c.properties rename to testsuite/performance/tests/parameters/datasets/2ku200c.properties diff --git a/testsuite/performance/tests/datasets/default.properties b/testsuite/performance/tests/parameters/datasets/2u2c.properties similarity index 100% rename from testsuite/performance/tests/datasets/default.properties rename to testsuite/performance/tests/parameters/datasets/2u2c.properties diff --git a/testsuite/performance/tests/datasets/500ku500c.properties b/testsuite/performance/tests/parameters/datasets/500ku500c.properties similarity index 100% rename from testsuite/performance/tests/datasets/500ku500c.properties rename to testsuite/performance/tests/parameters/datasets/500ku500c.properties diff --git a/testsuite/performance/tests/parameters/datasets/volume-testing/big.properties b/testsuite/performance/tests/parameters/datasets/volume-testing/big.properties new file mode 100644 index 0000000000..4919199241 --- /dev/null +++ b/testsuite/performance/tests/parameters/datasets/volume-testing/big.properties @@ -0,0 +1,8 @@ +numOfRealms=200 +usersPerRealm=1000000 +clientsPerRealm=2 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/parameters/datasets/volume-testing/medium.properties b/testsuite/performance/tests/parameters/datasets/volume-testing/medium.properties new file mode 100644 index 0000000000..b80e841a9b --- /dev/null +++ b/testsuite/performance/tests/parameters/datasets/volume-testing/medium.properties @@ -0,0 +1,8 @@ +numOfRealms=20 +usersPerRealm=10000 +clientsPerRealm=2 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/parameters/datasets/volume-testing/small.properties b/testsuite/performance/tests/parameters/datasets/volume-testing/small.properties new file mode 100644 index 0000000000..a281f9056c --- /dev/null +++ b/testsuite/performance/tests/parameters/datasets/volume-testing/small.properties @@ -0,0 +1,8 @@ +numOfRealms=2 +usersPerRealm=1000 +clientsPerRealm=2 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/cluster.properties b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/cluster.properties new file mode 100644 index 0000000000..d9b1c9d9db --- /dev/null +++ b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/cluster.properties @@ -0,0 +1,32 @@ +#provisioner=docker-compose +#deployment=cluster + +# Keycloak Settings +keycloak.scale=1 +keycloak.docker.cpusets=2 3 +keycloak.docker.memlimit=2500m +keycloak.jvm.memory=-Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m +keycloak.http.max-connections=50000 +keycloak.ajp.max-connections=50000 +keycloak.worker.io-threads=2 +keycloak.worker.task-max-threads=16 +keycloak.ds.min-pool-size=10 +keycloak.ds.max-pool-size=100 +keycloak.ds.pool-prefill=true +keycloak.ds.ps-cache-size=100 + +# Database Settings +db.docker.cpusets=1 +db.docker.memlimit=2g +db.max.connections=100 + +# Load Balancer Settings +lb.docker.cpusets=1 +lb.docker.memlimit=1500m +lb.jvm.memory=-Xms64m -Xmx1024m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m +lb.http.max-connections=50000 +lb.worker.io-threads=2 +lb.worker.task-max-threads=16 + +# Monitoring Settings +monitoring.docker.cpusets=0 diff --git a/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/crossdc.properties b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/crossdc.properties new file mode 100644 index 0000000000..4a5083f235 --- /dev/null +++ b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/crossdc.properties @@ -0,0 +1,42 @@ +#provisioner=docker-compose +#deployment=crossdc + +# Keycloak Settings +keycloak.dc1.scale=1 +keycloak.dc2.scale=1 +keycloak.dc1.docker.cpusets=2 +keycloak.dc2.docker.cpusets=3 +keycloak.docker.memlimit=2500m +keycloak.jvm.memory=-Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m +keycloak.http.max-connections=50000 +keycloak.ajp.max-connections=50000 +keycloak.worker.io-threads=2 +keycloak.worker.task-max-threads=16 +keycloak.ds.min-pool-size=10 +keycloak.ds.max-pool-size=100 +keycloak.ds.pool-prefill=true +keycloak.ds.ps-cache-size=100 + +# Database Settings +db.dc1.docker.cpusets=1 +db.dc2.docker.cpusets=1 +db.docker.memlimit=2g +db.max.connections=100 + +# Load Balancer Settings +lb.dc1.docker.cpusets=1 +lb.dc2.docker.cpusets=1 +lb.docker.memlimit=1500m +lb.jvm.memory=-Xms64m -Xmx1024m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m +lb.http.max-connections=50000 +lb.worker.io-threads=2 +lb.worker.task-max-threads=16 + +# Infinispan Settings +infinispan.dc1.docker.cpusets=1 +infinispan.dc2.docker.cpusets=1 +infinispan.docker.memlimit=1500m +infinispan.jvm.memory=-Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC + +# Monitoring Settings +monitoring.docker.cpusets=0 diff --git a/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/singlenode.properties b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/singlenode.properties new file mode 100644 index 0000000000..e4a52c0f7d --- /dev/null +++ b/testsuite/performance/tests/parameters/provisioning/docker-compose/4cpus/singlenode.properties @@ -0,0 +1,23 @@ +#provisioner=docker-compose +#deployment=singlenode + +# Keycloak Settings +keycloak.scale=1 +keycloak.docker.cpusets=2-3 +keycloak.docker.memlimit=2500m +keycloak.jvm.memory=-Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m +keycloak.http.max-connections=50000 +keycloak.worker.io-threads=2 +keycloak.worker.task-max-threads=16 +keycloak.ds.min-pool-size=10 +keycloak.ds.max-pool-size=100 +keycloak.ds.pool-prefill=true +keycloak.ds.ps-cache-size=100 + +# Database Settings +db.docker.cpusets=1 +db.docker.memlimit=2g +db.max.connections=100 + +# Monitoring Settings +monitoring.docker.cpusets=0 diff --git a/testsuite/performance/tests/parameters/test/admin-console.properties b/testsuite/performance/tests/parameters/test/admin-console.properties new file mode 100644 index 0000000000..c3a2773ebd --- /dev/null +++ b/testsuite/performance/tests/parameters/test/admin-console.properties @@ -0,0 +1,8 @@ +gatling.simulationClass=keycloak.AdminConsoleSimulation +usersPerSec=0.2 +rampUpPeriod=15 +warmUpPeriod=15 +measurementPeriod=30 +filterResults=false +userThinkTime=0 +refreshTokenPeriod=0 diff --git a/testsuite/performance/tests/parameters/test/basic-oidc.properties b/testsuite/performance/tests/parameters/test/basic-oidc.properties new file mode 100644 index 0000000000..c9a4af0d98 --- /dev/null +++ b/testsuite/performance/tests/parameters/test/basic-oidc.properties @@ -0,0 +1,10 @@ +gatling.simulationClass=keycloak.BasicOIDCSimulation +usersPerSec=1.0 +rampUpPeriod=15 +warmUpPeriod=15 +measurementPeriod=30 +filterResults=false +userThinkTime=0 +refreshTokenPeriod=0 +refreshTokenCount=1 +badLoginAttempts=1 diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml index 37bb52bf20..7cbc4bbee3 100644 --- a/testsuite/performance/tests/pom.xml +++ b/testsuite/performance/tests/pom.xml @@ -32,56 +32,20 @@ docker-compose singlenode + + ${provisioner}/4cpus/${deployment} + 2u2c + basic-oidc + ${project.basedir}/parameters/provisioning/${provisioning.properties}.properties + ${project.basedir}/parameters/datasets/${dataset}.properties + ${project.basedir}/parameters/test/${test.properties}.properties + ${project.build.directory}/provisioned-system.properties - - - - - 2-3 - 2 - 3 - 2500m - -Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m - 50000 - 50000 - 2 - 16 - 10 - 100 - true - 100 - - - 1 - 1 - 1 - 2g - 100 - https://downloads.jboss.org/keycloak-qe - - - 1 - 1 - 1 - 1500m - -Xms64m -Xmx1024m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m - 50000 - 2 - 16 - - - 1 - 1 - 1500m - -Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC - - - 0 - - - default + + + https://downloads.jboss.org/keycloak-qe/${server.version} 1 1.8 @@ -206,6 +170,20 @@ properties-maven-plugin 1.0.0 + + read-parameters + initialize + + read-project-properties + + + + ${provisioning.properties.file} + ${dataset.properties.file} + ${test.properties.file} + + + read-existing-provisioned-system-properties initialize @@ -274,12 +252,11 @@ ${gatling.skip.run} true true - + -Dproject.build.directory=${project.build.directory} -Dkeycloak.server.uris=${keycloak.frontend.servers} + -DnumOfRealms=${numOfRealms} -DusersPerRealm=${usersPerRealm} -DclientsPerRealm=${clientsPerRealm} @@ -288,6 +265,16 @@ -DclientRolesPerUser=${clientRolesPerUser} -DclientRolesPerClient=${clientRolesPerClient} -DhashIterations=${hashIterations} + + -DusersPerSec=${usersPerSec} + -DrampUpPeriod=${rampUpPeriod} + -DwarmUpPeriod=${warmUpPeriod} + -DmeasurementPeriod=${measurementPeriod} + -DfilterResults=${filterResults} + -DuserThinkTime=${userThinkTime} + -DrefreshTokenPeriod=${refreshTokenPeriod} + -DrefreshTokenCount=${refreshTokenCount} + -DbadLoginAttempts=${badLoginAttempts} @@ -313,59 +300,10 @@ - - docker-compose - - - !provisioner - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - copy-dockerfiles-etc - generate-resources - - run - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - cluster cluster - 2 3 @@ -377,8 +315,30 @@ provision + + ${project.basedir} + ${project.build.directory} + + + org.apache.maven.plugins + maven-antrun-plugin + + + prepare-provisioning + generate-resources + + run + + + + + + + + + org.codehaus.mojo exec-maven-plugin @@ -465,39 +425,6 @@ - - initialize-dataset-properties - - - dataset - - - - - - org.codehaus.mojo - properties-maven-plugin - 1.0.0 - - - initialize-dataset-properties - pre-integration-test - - read-project-properties - - - - ${project.basedir}/datasets/${dataset}.properties - - true - - - - - - - - generate-data @@ -559,26 +486,6 @@ export-dump - - maven-enforcer-plugin - - - enforce-nondefault-dataset - - enforce - - - - - dataset - (?!default).* - For the "export-dump" task property "dataset" cannot be set to "default". - - - - - - org.codehaus.mojo exec-maven-plugin @@ -595,7 +502,7 @@ ${provisioner} ${deployment} export-dump - ${dataset} + ${dataset.properties.file} @@ -625,7 +532,7 @@ ${provisioner} ${deployment} import-dump - ${dataset} + ${dataset.properties.file} ${db.dump.download.site} diff --git a/testsuite/performance/tests/prepare-provisioning.xml b/testsuite/performance/tests/prepare-provisioning.xml new file mode 100644 index 0000000000..3108bac337 --- /dev/null +++ b/testsuite/performance/tests/prepare-provisioning.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java index 288b06768a..6d2f19461d 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java @@ -58,9 +58,7 @@ public class TestConfig { public static final int rampUpPeriod = Integer.getInteger("rampUpPeriod", 0); public static final int warmUpPeriod = Integer.getInteger("warmUpPeriod", 0); public static final int measurementPeriod = Integer.getInteger("measurementPeriod", 30); - public static final boolean rampDownASAP = Boolean.getBoolean("rampDownASAP"); // check for rampdown condition after each scenario step public static final boolean filterResults = Boolean.getBoolean("filterResults"); // filter out results outside of measurementPeriod - public static final int pace = Integer.getInteger("pace", 0); // additional dynamic "pause buffer" between scenario loops public static final int userThinkTime = Integer.getInteger("userThinkTime", 0); public static final int refreshTokenPeriod = Integer.getInteger("refreshTokenPeriod", 0);