diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index 6884e4ab13..e229824395 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -135,26 +135,30 @@ For example: `mvn verify -Ptest -DrunUsers=1 -DnumOfIterations=10 -DuserThinkTime=0 -Ddataset=100u -DrefreshTokenPeriod=10 -Dgatling.simulationClass=keycloak.AdminSimulation` -## Debugging & Profiling +## Monitoring -Keycloak docker container exposes JMX management interface on port `9990`. +### JMX -### JVisualVM +To enable access to JMX on the WildFly-backed services set properties `management.user` and `management.user.password` during the provisioning phase. +#### JVisualVM + +- Set `JBOSS_HOME` variable to point to a valid WildFly 10+ installation. - Start JVisualVM with `jboss-client.jar` on classpath: `./jvisualvm --cp:a $JBOSS_HOME/bin/client/jboss-client.jar`. -- Add a local JMX connection: `service:jmx:remote+http://localhost:9990`. +- Add a local JMX connection: `service:jmx:remote+http://localhost:9990`. **[*]** - Check "Use security credentials" and set `admin:admin`. (The default credentials can be overriden by providing env. variables `DEBUG_USER` and `DEBUG_USER_PASSWORD` to the container.) - Open the added connection. -_Note: The above applies for the singlenode deployment. -In cluster/crossdc deployments there are multiple KC containers running at the same time so their exposed ports are mapped to random available ports on `0.0.0.0`. -To find the actual mapped ports run command: `docker ps | grep performance_keycloak`._ +**[*]** For `singlenode` this points to the JMX console of the Keycloak server. +To get the connection URLs for `cluster` or `crossdc` deployments see the JMX section in the generated `provisioned-system.properties` file. +- Property `keycloak.frontend.servers.jmx` contains JMX URLs of the Load Balancers. +- Property `keycloak.backend.servers.jmx` contains JMX URLs of the clustered Keycloak servers. +- Property `infinispan.servers.jmx` contains JMX URLs of the Infinispan servers, in Cross-DC deployment. +### Docker Monitoring -## Monitoring - -There is a docker-based solution for monitoring of CPU, memory and network usage per container. -(It uses CAdvisor service to export container metrics into InfluxDB time series database, and Grafana web app to query the DB and present results as graphs.) +There is a docker-based solution for monitoring CPU, memory and network usage per container. +It uses CAdvisor service to export container metrics into InfluxDB time series database, and Grafana web app to query the DB and present results as graphs. - To enable run: `mvn verify -Pmonitoring` - To disable run: `mvn verify -Pmonitoring-off[,delete-monitoring-data]`. diff --git a/testsuite/performance/README.provisioning-parameters.md b/testsuite/performance/README.provisioning-parameters.md index b291159099..3ce081a417 100644 --- a/testsuite/performance/README.provisioning-parameters.md +++ b/testsuite/performance/README.provisioning-parameters.md @@ -82,6 +82,13 @@ The maximum cluster size corresponds to the number of cpusets. | Category | Setting | Property | Default Value | |-------------|-------------------------------|-----------------------------|-----------------| | Docker | Allocated CPUs | `monitoring.docker.cpusets` | `0` | +| JMX | Management user | `management.user` | Not set. | +| | Management user's password | `management.user.password` | Not set. | + +By setting the `managemen.user` and `management.user.password` parameters it is possible +to add a management user to all WildFly-backed services (*Keycloak Server*, *Infinispan Server* and the *Load Balancer*). +Unless both parameters are explicitly provided during the provisioning phase the user will not be added +and it won't be possible to log into the management console or access JMX. ## Note on Docker settings diff --git a/testsuite/performance/infinispan/Dockerfile b/testsuite/performance/infinispan/Dockerfile index d151ac735b..4b61e2aaba 100644 --- a/testsuite/performance/infinispan/Dockerfile +++ b/testsuite/performance/infinispan/Dockerfile @@ -4,6 +4,9 @@ FROM jboss/infinispan-server:8.2.6.Final ARG LOCAL_SITE ARG REMOTE_SITE +ARG MANAGEMENT_USER +ARG MANAGEMENT_USER_PASS + USER root RUN yum -y install iproute USER jboss @@ -19,6 +22,7 @@ USER root RUN chmod -v +x /usr/local/bin/*.sh USER jboss +RUN if [ ! -z "$MANAGEMENT_USER" ]; then $INFINISPAN_SERVER_HOME/bin/add-user.sh -u $MANAGEMENT_USER -p $MANAGEMENT_USER_PASS ; fi RUN $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-private-network-interface.cli; \ $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-keycloak-caches.cli; \ cd $INFINISPAN_SERVER_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp diff --git a/testsuite/performance/keycloak/configure.xml b/testsuite/performance/keycloak/configure.xml index b6572ff0d2..6bd3fc9763 100644 --- a/testsuite/performance/keycloak/configure.xml +++ b/testsuite/performance/keycloak/configure.xml @@ -1,12 +1,16 @@ - + + + performance.configured: ${performance.configured} + management.configured: ${management.configured} + crossdc.configured: ${crossdc.configured} - - keycloak-performance-configuration + + Applying keycloak performance configuration. @@ -23,41 +27,53 @@ + + + + - - - - - - + + + + - - - - crossdc.configured: ${crossdc.configured} - + + Adding management user: `${management.user}` + + + + + + + + + - + keycloak-crossdc-configuration + + + + diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml index 1c16748bd5..eb189a7c35 100644 --- a/testsuite/performance/keycloak/pom.xml +++ b/testsuite/performance/keycloak/pom.xml @@ -51,11 +51,9 @@ true false + true false - admin - admin - ${project.build.scriptSourceDirectory} ${project.basedir}/src/main/resources @@ -135,6 +133,19 @@ + + add-management-user + process-resources + + run + + + ${skip.add.management.user} + + + + + keycloak-docker process-resources @@ -167,6 +178,22 @@ + + add-management-user + + + management.user + + + + false + + ${management.user} + ${management.user.password} + + + crossdc @@ -185,4 +212,4 @@ - \ No newline at end of file + diff --git a/testsuite/performance/pom.xml b/testsuite/performance/pom.xml index 004757ef89..68f6cd0054 100644 --- a/testsuite/performance/pom.xml +++ b/testsuite/performance/pom.xml @@ -31,6 +31,11 @@ Keycloak Performance TestSuite pom + + + + + keycloak tests diff --git a/testsuite/performance/tests/docker-compose.sh b/testsuite/performance/tests/docker-compose.sh index 0700fca7a2..081be731e7 100755 --- a/testsuite/performance/tests/docker-compose.sh +++ b/testsuite/performance/tests/docker-compose.sh @@ -52,13 +52,20 @@ function inspectDockerPortMapping() { function generateProvisionedSystemProperties() { echo "Generating $PROVISIONED_SYSTEM_PROPERTIES_FILE" echo "deployment=$DEPLOYMENT" > $PROVISIONED_SYSTEM_PROPERTIES_FILE + echo "# Docker Compose" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE echo "keycloak.docker.services=$KEYCLOAK_SERVICES" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE case "$DEPLOYMENT" in singlenode) + echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_keycloak_1 echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + + echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_keycloak_1 + echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$MAPPED_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE ;; cluster) + echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_1 echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE BACKEND_URLS="" @@ -67,8 +74,19 @@ function generateProvisionedSystemProperties() { BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth" done echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + + echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_1 + echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$MAPPED_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + BACKEND_URLS="" + for SERVICE in $KEYCLOAK_SERVICES ; do + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_${SERVICE}_1 + BACKEND_URLS="$BACKEND_URLS service:jmx:remote+http://localhost:$MAPPED_PORT" + done + echo "keycloak.backend.servers.jmx=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE ;; crossdc) + echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc1_1 KC_DC1_PORT=$MAPPED_PORT inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc2_1 @@ -80,6 +98,25 @@ function generateProvisionedSystemProperties() { BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth" done echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + + echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_dc1_1 + KC_DC1_PORT=$MAPPED_PORT + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_dc2_1 + KC_DC2_PORT=$MAPPED_PORT + echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$KC_DC1_PORT service:jmx:remote+http://localhost:$KC_DC2_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + BACKEND_URLS="" + for SERVICE in $KEYCLOAK_SERVICES ; do + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_${SERVICE}_1 + BACKEND_URLS="$BACKEND_URLS service:jmx:remote+http://localhost:$MAPPED_PORT" + done + echo "keycloak.backend.servers.jmx=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE + + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_infinispan_dc1_1 + ISPN_DC1_PORT=$MAPPED_PORT + inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_infinispan_dc2_1 + ISPN_DC2_PORT=$MAPPED_PORT + echo "infinispan.servers.jmx=service:jmx:remote+http://localhost:$ISPN_DC1_PORT service:jmx:remote+http://localhost:$ISPN_DC2_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE ;; esac } diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml index 818f8c95be..505db68c06 100644 --- a/testsuite/performance/tests/pom.xml +++ b/testsuite/performance/tests/pom.xml @@ -391,6 +391,9 @@ ${project.version} + ${management.user} + ${management.user.password} + ${keycloak.scale} ${keycloak.dc1.scale} ${keycloak.dc2.scale} diff --git a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml index 7d152c7d4b..4b8cb7c82d 100644 --- a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml +++ b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml @@ -47,5 +47,6 @@ services: WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} ports: - "8080:8080" + - "9990:9990" diff --git a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml index e51e192dcb..f92062eeb4 100644 --- a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml +++ b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml @@ -35,6 +35,8 @@ services: args: LOCAL_SITE: dc1 REMOTE_SITE: dc2 + MANAGEMENT_USER: ${MANAGEMENT_USER} + MANAGEMENT_USER_PASS: ${MANAGEMENT_USER_PASS} image: keycloak_test_infinispan_dc1:${KEYCLOAK_VERSION:-latest} cpuset: ${INFINISPAN_DC1_CPUSET:-1} mem_limit: ${INFINISPAN_MEMLIMIT:-1500m} @@ -49,7 +51,7 @@ services: TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600] JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true ports: - - "9991:9990" + - "9990" infinispan_dc2: build: @@ -57,6 +59,8 @@ services: args: LOCAL_SITE: dc2 REMOTE_SITE: dc1 + MANAGEMENT_USER: ${MANAGEMENT_USER} + MANAGEMENT_USER_PASS: ${MANAGEMENT_USER_PASS} image: keycloak_test_infinispan_dc2:${KEYCLOAK_VERSION:-latest} depends_on: infinispan_dc1: @@ -74,7 +78,7 @@ services: TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600] JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true ports: - - "9992:9990" + - "9990" mariadb_dc1: @@ -94,7 +98,7 @@ services: entrypoint: docker-entrypoint-wsrep.sh command: --wsrep-new-cluster ports: - - "3306:3306" + - "3307:3306" mariadb_dc2: build: db/mariadb @@ -113,7 +117,7 @@ services: entrypoint: docker-entrypoint-wsrep.sh command: --wsrep_cluster_address=gcomm://mariadb_dc1 ports: - - "3307:3306" + - "3308:3306" loadbalancer_dc1: @@ -133,6 +137,7 @@ services: WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} ports: - "8081:8080" + - "9991:9990" loadbalancer_dc2: build: load-balancer/wildfly-modcluster @@ -151,4 +156,5 @@ services: WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16} ports: - "8082:8080" + - "9992:9990"