diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index ba8cc305da..3eadbccef0 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -70,7 +70,7 @@ Provisioning/teardown is performed via `docker-compose` tool. More details in [R ### Generate Test Data -Usage: `mvn verify -Pgenerate-data[,cluster] [-Ddataset=DATASET] [-D=]`. +Usage: `mvn verify -Pgenerate-data[,cluster] [-Ddataset=DATASET] [-Dexport-dump] [-D=]`. Dataset properties are loaded from `datasets/${dataset}.properties` file. Individual properties can be overriden by specifying `-D` params. @@ -85,8 +85,9 @@ Dataset data is first generated as a .json file, and then imported into Keycloak The data can also be exported from the database, and stored locally as `datasets/${dataset}.sql.gz` `DATASET=100u ./prepare-dump.sh` -If there is a data dump file available then -Pimport-dump can be used to import the data directly into the database, -by-passing Keycloak server completely. +To speed up dataset initialization part, it is possible to pass `-Dexport-dump` option to have the generated dataset +exported right after it has been generated. Then, if there is a data dump file available then `-Pimport-dump` +can be used to import the data directly into the database, bypassing Keycloak server completely. Usage: `mvn verify -Pimport-dump [-Ddataset=DATASET]` diff --git a/testsuite/performance/README.provisioning-parameters.md b/testsuite/performance/README.provisioning-parameters.md index bee8ba41a0..eaea691f06 100644 --- a/testsuite/performance/README.provisioning-parameters.md +++ b/testsuite/performance/README.provisioning-parameters.md @@ -4,7 +4,7 @@ | Category | Setting | Property | Default value | |-------------|-------------------------------|------------------------------------|------------------------------------------------------------------| -| JVM | Memory settings | `keycloak.jvm.memory` | -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m | +| JVM | Memory settings | `keycloak.jvm.memory` | -Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m | | Undertow | HTTP Listener max connections | `keycloak.http.max-connections` | 500 | | | AJP Listener max connections | `keycloak.ajp.max-connections` | 500 | | IO | Worker IO thread pool | `keycloak.worker.io-threads` | 2 | @@ -29,35 +29,20 @@ |-------------|-------------------------------|-------------------------|-----------------------------------------------------------------------------------------| | JVM | Memory settings | `infinispan.jvm.memory` | -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC | -## CPUs +## Docker settings -At the moment it is not possible to dynamically parametrize the number of CPUs for a service via Maven properties or environment variables. +By default, there are 4 CPU cores allocated: core 0 for monitoring, core 1 for database (MariaDB), and cores 2 and 3 for Keycloak server. +Default memory limits for database and Keycloak server are 2g. The `cpuset` and `memlimit` parameters set here are set to `cpuset` and +`mem_limit` parameters of docker-compose configuration. See docker-compose documentation for meaning of the values. How to set the parameters +correctly depends on number of factors - number of cpu cores, NUMA, available memory etc., hence it is out of scope of this document. -To change the default value (`cpus: 1`) it is necessary to edit the Docker Compose file. +| Container | Setting | Property | Default value | +|-------------|-------------------------------|---------------------------------|-------------------------------------------------------| +| Keycloak | Allocated CPUs | `keycloak.docker.cpuset` | 2-3 | +| | Allocated CPUs for DC1 | `keycloak.dc1.docker.cpuset` | 2-3 | +| | Allocated CPUs for DC2 | `keycloak.dc2.docker.cpuset` | 2-3 | +| | Available memory | `keycloak.docker.memlimit` | 2g | +| MariaDB | Allocated CPUs | `db.docker.cpuset` | 1 | +| | Available memory | `db.docker.memlimit` | 2g | +| Monitoring | Allocated CPUs | `monitoring.docker.cpuset` | 0 | - -### Example: Keycloak service using 2 CPU cores - -`docker-compose.yml` and `docker-compose-cluster.yml`: -``` -services: - ... - keycloak: - ... - cpus: 2 - ... -``` - -`docker-compose-crossdc.yml`: -``` -services: - ... - keycloak_dc1: - ... - cpus: 2 - ... - keycloak_dc2: - ... - cpus: 2 - ... -``` diff --git a/testsuite/performance/docker-compose-cluster.yml b/testsuite/performance/docker-compose-cluster.yml index d04bddd69d..4ecd95aaf6 100644 --- a/testsuite/performance/docker-compose-cluster.yml +++ b/testsuite/performance/docker-compose-cluster.yml @@ -15,7 +15,8 @@ services: mariadb: build: db/mariadb image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} - cpus: 1 + cpuset: ${DB_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-1g} networks: - keycloak environment: @@ -32,7 +33,8 @@ services: depends_on: mariadb: condition: service_healthy - cpus: 1 + cpuset: ${KEYCLOAK_CPUSET:-2-3} + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} networks: - keycloak environment: @@ -46,7 +48,7 @@ services: KEYCLOAK_USER: admin KEYCLOAK_PASSWORD: admin - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} diff --git a/testsuite/performance/docker-compose-crossdc.yml b/testsuite/performance/docker-compose-crossdc.yml index e95f71dee6..5075ad999f 100644 --- a/testsuite/performance/docker-compose-crossdc.yml +++ b/testsuite/performance/docker-compose-crossdc.yml @@ -95,7 +95,8 @@ services: depends_on: mariadb_dc1: condition: service_healthy - cpus: 1 + cpuset: ${DB_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-1g} networks: - db_replication - dc2_keycloak @@ -122,7 +123,8 @@ services: # wait for the ispn cluster to be ready before starting keycloak infinispan_dc2: condition: service_healthy - cpus: 1 + cpuset: ${KEYCLOAK_DC1_CPUSET:-2} + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} networks: - dc1_keycloak environment: @@ -138,7 +140,7 @@ services: INFINISPAN_HOST: infinispan_dc1 SITE: dc1 - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} @@ -162,7 +164,8 @@ services: # wait for first kc instance to be ready before starting another keycloak_dc1: condition: service_healthy - cpus: 1 + cpuset: ${KEYCLOAK_DC2_CPUSET:-3} + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} networks: - dc2_keycloak environment: @@ -176,7 +179,7 @@ services: INFINISPAN_HOST: infinispan_dc2 SITE: dc2 - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} AJP_MAX_CONNECTIONS: ${KEYCLOAK_AJP_MAX_CONNECTIONS:-500} WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} diff --git a/testsuite/performance/docker-compose-monitoring.yml b/testsuite/performance/docker-compose-monitoring.yml index a93d55a266..308c11387a 100644 --- a/testsuite/performance/docker-compose-monitoring.yml +++ b/testsuite/performance/docker-compose-monitoring.yml @@ -11,6 +11,7 @@ services: monitoring_influxdb: image: influxdb + cpuset: ${MONITORING_CPUSET:-1} volumes: - influx:/var/lib/influxdb networks: @@ -26,6 +27,7 @@ services: monitoring_cadvisor: build: monitoring/cadvisor image: monitoring_cadvisor + cpuset: ${MONITORING_CPUSET:-1} hostname: '{{.Node.ID}}' volumes: - /:/rootfs:ro @@ -50,6 +52,7 @@ services: monitoring_grafana: build: monitoring/grafana image: monitoring_grafana + cpuset: ${MONITORING_CPUSET:-1} depends_on: - monitoring_influxdb volumes: diff --git a/testsuite/performance/docker-compose.yml b/testsuite/performance/docker-compose.yml index 0ab4ae08c5..503f0d4191 100644 --- a/testsuite/performance/docker-compose.yml +++ b/testsuite/performance/docker-compose.yml @@ -5,13 +5,14 @@ networks: ipam: config: - subnet: 10.0.1.0/24 - + services: - + mariadb: build: db/mariadb image: keycloak_test_mariadb:${KEYCLOAK_VERSION:-latest} - cpus: 1 + cpuset: ${DB_CPUSET:-1} + mem_limit: ${DB_MEMLIMIT:-1g} networks: - keycloak environment: @@ -22,14 +23,15 @@ services: MYSQL_INITDB_SKIP_TZINFO: 1 ports: - "3306:3306" - + keycloak: build: keycloak image: keycloak_test_keycloak:${KEYCLOAK_VERSION:-latest} depends_on: mariadb: condition: service_healthy - cpus: 1 + cpuset: ${KEYCLOAK_CPUSET:-2-3} + mem_limit: ${KEYCLOAK_MEMLIMIT:-2500m} networks: - keycloak environment: @@ -40,7 +42,7 @@ services: KEYCLOAK_USER: admin KEYCLOAK_PASSWORD: admin # docker-compose syntax note: ${ENV_VAR:-} - JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true + JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-500} WORKER_IO_THREADS: ${KEYCLOAK_WORKER_IO_THREADS:-2} WORKER_TASK_MAX_THREADS: ${KEYCLOAK_WORKER_TASK_MAX_THREADS:-16} @@ -50,4 +52,4 @@ services: DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100} ports: - "8080:8080" - - "9990:9990" \ No newline at end of file + - "9990:9990" diff --git a/testsuite/performance/keycloak/Dockerfile b/testsuite/performance/keycloak/Dockerfile index 487bad9900..91866e82c9 100644 --- a/testsuite/performance/keycloak/Dockerfile +++ b/testsuite/performance/keycloak/Dockerfile @@ -17,26 +17,23 @@ ADD target/keycloak configs/ ./ ADD *.sh /usr/local/bin/ USER root -RUN chown -R jboss .; chgrp -R jboss .; -RUN chmod -R -v +x /usr/local/bin/ -RUN yum install -y epel-release jq iproute && yum clean all +RUN chown -R jboss .; chgrp -R jboss .; \ + chmod -R -v +x /usr/local/bin/ ; \ + yum install -y epel-release jq iproute && yum clean all USER jboss # install mariadb JDBC driver -RUN mkdir -p modules/system/layers/base/org/mariadb/jdbc/main; \ - cd modules/system/layers/base/org/mariadb/jdbc/main; \ - curl -O http://central.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.0.3/mariadb-java-client-2.0.3.jar -ADD module.xml modules/system/layers/base/org/mariadb/jdbc/main/ -# apply configurations -RUN $JBOSS_HOME/bin/jboss-cli.sh --file=set-keycloak-ds.cli -RUN $JBOSS_HOME/bin/jboss-cli.sh --file=io-worker-threads.cli -RUN $JBOSS_HOME/bin/jboss-cli.sh --file=undertow.cli -RUN $JBOSS_HOME/bin/jboss-cli.sh --file=distributed-cache-owners.cli -RUN $JBOSS_HOME/bin/jboss-cli.sh --file=modcluster-simple-load-provider.cli -RUN if [ "$REMOTE_CACHES" == "true" ]; then $JBOSS_HOME/bin/jboss-cli.sh --file=add-remote-cache-stores.cli; fi -RUN cd $JBOSS_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp +RUN curl --create-dirs --output modules/system/layers/base/org/mariadb/jdbc/main/mariadb-java-client-2.0.3.jar http://central.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/2.0.3/mariadb-java-client-2.0.3.jar ; \ + $JBOSS_HOME/bin/jboss-cli.sh --file=set-keycloak-ds.cli && \ + $JBOSS_HOME/bin/jboss-cli.sh --file=io-worker-threads.cli && \ + $JBOSS_HOME/bin/jboss-cli.sh --file=undertow.cli && \ + $JBOSS_HOME/bin/jboss-cli.sh --file=distributed-cache-owners.cli && \ + $JBOSS_HOME/bin/jboss-cli.sh --file=modcluster-simple-load-provider.cli && \ + if [ "$REMOTE_CACHES" == "true" ]; then $JBOSS_HOME/bin/jboss-cli.sh --file=add-remote-cache-stores.cli; fi && \ + cd $JBOSS_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp ; \ + $JBOSS_HOME/bin/add-user.sh -u $DEBUG_USER -p $DEBUG_USER_PASSWORD -RUN $JBOSS_HOME/bin/add-user.sh -u $DEBUG_USER -p $DEBUG_USER_PASSWORD +ADD module.xml modules/system/layers/base/org/mariadb/jdbc/main/ EXPOSE 8080 EXPOSE 9990 diff --git a/testsuite/performance/keycloak/docker-entrypoint.sh b/testsuite/performance/keycloak/docker-entrypoint.sh index 9728f98314..eb44c09a8a 100644 --- a/testsuite/performance/keycloak/docker-entrypoint.sh +++ b/testsuite/performance/keycloak/docker-entrypoint.sh @@ -15,4 +15,3 @@ if [ $KEYCLOAK_USER ] && [ $KEYCLOAK_PASSWORD ]; then fi exec /opt/jboss/keycloak/bin/standalone.sh $PARAMS -exit $? diff --git a/testsuite/performance/prepare-dump.sh b/testsuite/performance/prepare-dump.sh index 9034e39b59..23bd99c9ce 100755 --- a/testsuite/performance/prepare-dump.sh +++ b/testsuite/performance/prepare-dump.sh @@ -4,13 +4,7 @@ GATLING_HOME=$DIRNAME/tests if [ -z "$DATASET" ]; then echo "This script requires DATASET env variable to be set" - echo 1 -fi - -./prepare-data.sh $@ -if [ $? -ne 0 ]; then - echo "Failed! See log file for details." - exit $? + exit 1 fi echo "Exporting dump file" diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml index f0414dbaf4..150c5fdc83 100644 --- a/testsuite/performance/tests/pom.xml +++ b/testsuite/performance/tests/pom.xml @@ -39,7 +39,7 @@ http://localhost:8080/auth jdbc:mariadb://keycloak:keycloak@localhost:3306/keycloak - -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m + -Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m 500 500 2 @@ -48,14 +48,25 @@ 100 true 100 - + -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m 500 2 16 - + + + 1 + 2-3 + 2 + 3 + 0 + + 2g + 2g + + -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC - + default 1 @@ -321,6 +332,13 @@ -f ${compose.file} up -d --build ${compose.up.params} ${project.version} + + ${keycloak.docker.cpuset} + ${keycloak.dc1.docker.cpuset} + ${keycloak.dc2.docker.cpuset} + ${keycloak.docker.memlimit} + ${db.docker.cpuset} + ${db.docker.memlimit} ${keycloak.jvm.memory} ${keycloak.http.max-connections} @@ -462,6 +480,43 @@ + + export-dump-after-generation + + + false + + export-dump + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + export-dump + pre-integration-test + + exec + + + ${project.basedir}/.. + ./prepare-dump.sh + + + ${dataset} + + + + + + + + + import-dump @@ -581,6 +636,9 @@ ${project.basedir}/.. docker-compose -f docker-compose-monitoring.yml up -d --build + + ${monitoring.docker.cpuset} +