diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md
index 6884e4ab13..e229824395 100644
--- a/testsuite/performance/README.md
+++ b/testsuite/performance/README.md
@@ -135,26 +135,30 @@ For example:
`mvn verify -Ptest -DrunUsers=1 -DnumOfIterations=10 -DuserThinkTime=0 -Ddataset=100u -DrefreshTokenPeriod=10 -Dgatling.simulationClass=keycloak.AdminSimulation`
-## Debugging & Profiling
+## Monitoring
-Keycloak docker container exposes JMX management interface on port `9990`.
+### JMX
-### JVisualVM
+To enable access to JMX on the WildFly-backed services set properties `management.user` and `management.user.password` during the provisioning phase.
+#### JVisualVM
+
+- Set `JBOSS_HOME` variable to point to a valid WildFly 10+ installation.
- Start JVisualVM with `jboss-client.jar` on classpath: `./jvisualvm --cp:a $JBOSS_HOME/bin/client/jboss-client.jar`.
-- Add a local JMX connection: `service:jmx:remote+http://localhost:9990`.
+- Add a local JMX connection: `service:jmx:remote+http://localhost:9990`. **[*]**
- Check "Use security credentials" and set `admin:admin`. (The default credentials can be overriden by providing env. variables `DEBUG_USER` and `DEBUG_USER_PASSWORD` to the container.)
- Open the added connection.
-_Note: The above applies for the singlenode deployment.
-In cluster/crossdc deployments there are multiple KC containers running at the same time so their exposed ports are mapped to random available ports on `0.0.0.0`.
-To find the actual mapped ports run command: `docker ps | grep performance_keycloak`._
+**[*]** For `singlenode` this points to the JMX console of the Keycloak server.
+To get the connection URLs for `cluster` or `crossdc` deployments see the JMX section in the generated `provisioned-system.properties` file.
+- Property `keycloak.frontend.servers.jmx` contains JMX URLs of the Load Balancers.
+- Property `keycloak.backend.servers.jmx` contains JMX URLs of the clustered Keycloak servers.
+- Property `infinispan.servers.jmx` contains JMX URLs of the Infinispan servers, in Cross-DC deployment.
+### Docker Monitoring
-## Monitoring
-
-There is a docker-based solution for monitoring of CPU, memory and network usage per container.
-(It uses CAdvisor service to export container metrics into InfluxDB time series database, and Grafana web app to query the DB and present results as graphs.)
+There is a docker-based solution for monitoring CPU, memory and network usage per container.
+It uses CAdvisor service to export container metrics into InfluxDB time series database, and Grafana web app to query the DB and present results as graphs.
- To enable run: `mvn verify -Pmonitoring`
- To disable run: `mvn verify -Pmonitoring-off[,delete-monitoring-data]`.
diff --git a/testsuite/performance/README.provisioning-parameters.md b/testsuite/performance/README.provisioning-parameters.md
index b291159099..3ce081a417 100644
--- a/testsuite/performance/README.provisioning-parameters.md
+++ b/testsuite/performance/README.provisioning-parameters.md
@@ -82,6 +82,13 @@ The maximum cluster size corresponds to the number of cpusets.
| Category | Setting | Property | Default Value |
|-------------|-------------------------------|-----------------------------|-----------------|
| Docker | Allocated CPUs | `monitoring.docker.cpusets` | `0` |
+| JMX | Management user | `management.user` | Not set. |
+| | Management user's password | `management.user.password` | Not set. |
+
+By setting the `managemen.user` and `management.user.password` parameters it is possible
+to add a management user to all WildFly-backed services (*Keycloak Server*, *Infinispan Server* and the *Load Balancer*).
+Unless both parameters are explicitly provided during the provisioning phase the user will not be added
+and it won't be possible to log into the management console or access JMX.
## Note on Docker settings
diff --git a/testsuite/performance/infinispan/Dockerfile b/testsuite/performance/infinispan/Dockerfile
index d151ac735b..4b61e2aaba 100644
--- a/testsuite/performance/infinispan/Dockerfile
+++ b/testsuite/performance/infinispan/Dockerfile
@@ -4,6 +4,9 @@ FROM jboss/infinispan-server:8.2.6.Final
ARG LOCAL_SITE
ARG REMOTE_SITE
+ARG MANAGEMENT_USER
+ARG MANAGEMENT_USER_PASS
+
USER root
RUN yum -y install iproute
USER jboss
@@ -19,6 +22,7 @@ USER root
RUN chmod -v +x /usr/local/bin/*.sh
USER jboss
+RUN if [ ! -z "$MANAGEMENT_USER" ]; then $INFINISPAN_SERVER_HOME/bin/add-user.sh -u $MANAGEMENT_USER -p $MANAGEMENT_USER_PASS ; fi
RUN $INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-private-network-interface.cli; \
$INFINISPAN_SERVER_HOME/bin/ispn-cli.sh --file=add-keycloak-caches.cli; \
cd $INFINISPAN_SERVER_HOME/standalone; rm -rf configuration/standalone_xml_history log data tmp
diff --git a/testsuite/performance/keycloak/configure.xml b/testsuite/performance/keycloak/configure.xml
index b6572ff0d2..6bd3fc9763 100644
--- a/testsuite/performance/keycloak/configure.xml
+++ b/testsuite/performance/keycloak/configure.xml
@@ -1,12 +1,16 @@
-
+
+
+
performance.configured: ${performance.configured}
+ management.configured: ${management.configured}
+ crossdc.configured: ${crossdc.configured}
-
- keycloak-performance-configuration
+
+ Applying keycloak performance configuration.
@@ -23,41 +27,53 @@
+
+
+
+
-
-
-
-
-
-
+
+
+
+
-
-
-
- crossdc.configured: ${crossdc.configured}
-
+
+ Adding management user: `${management.user}`
+
+
+
+
+
+
+
+
+
-
+
keycloak-crossdc-configuration
+
+
+
+
diff --git a/testsuite/performance/keycloak/pom.xml b/testsuite/performance/keycloak/pom.xml
index 1c16748bd5..eb189a7c35 100644
--- a/testsuite/performance/keycloak/pom.xml
+++ b/testsuite/performance/keycloak/pom.xml
@@ -51,11 +51,9 @@
true
false
+ true
false
- admin
- admin
-
${project.build.scriptSourceDirectory}
${project.basedir}/src/main/resources
@@ -135,6 +133,19 @@
+
+ add-management-user
+ process-resources
+
+ run
+
+
+ ${skip.add.management.user}
+
+
+
+
+
keycloak-docker
process-resources
@@ -167,6 +178,22 @@
+
+ add-management-user
+
+
+ management.user
+
+
+
+ false
+
+ ${management.user}
+ ${management.user.password}
+
+
+
crossdc
@@ -185,4 +212,4 @@
-
\ No newline at end of file
+
diff --git a/testsuite/performance/pom.xml b/testsuite/performance/pom.xml
index 004757ef89..68f6cd0054 100644
--- a/testsuite/performance/pom.xml
+++ b/testsuite/performance/pom.xml
@@ -31,6 +31,11 @@
Keycloak Performance TestSuite
pom
+
+
+
+
+
keycloak
tests
diff --git a/testsuite/performance/tests/docker-compose.sh b/testsuite/performance/tests/docker-compose.sh
index 0700fca7a2..081be731e7 100755
--- a/testsuite/performance/tests/docker-compose.sh
+++ b/testsuite/performance/tests/docker-compose.sh
@@ -52,13 +52,20 @@ function inspectDockerPortMapping() {
function generateProvisionedSystemProperties() {
echo "Generating $PROVISIONED_SYSTEM_PROPERTIES_FILE"
echo "deployment=$DEPLOYMENT" > $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ echo "# Docker Compose" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
echo "keycloak.docker.services=$KEYCLOAK_SERVICES" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
case "$DEPLOYMENT" in
singlenode)
+ echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_keycloak_1
echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+
+ echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_keycloak_1
+ echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$MAPPED_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
;;
cluster)
+ echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_1
echo "keycloak.frontend.servers=http://localhost:$MAPPED_PORT/auth" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
BACKEND_URLS=""
@@ -67,8 +74,19 @@ function generateProvisionedSystemProperties() {
BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth"
done
echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+
+ echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_1
+ echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$MAPPED_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ BACKEND_URLS=""
+ for SERVICE in $KEYCLOAK_SERVICES ; do
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_${SERVICE}_1
+ BACKEND_URLS="$BACKEND_URLS service:jmx:remote+http://localhost:$MAPPED_PORT"
+ done
+ echo "keycloak.backend.servers.jmx=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
;;
crossdc)
+ echo "# HTTP" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc1_1
KC_DC1_PORT=$MAPPED_PORT
inspectDockerPortMapping 8080/tcp ${PROJECT_NAME}_loadbalancer_dc2_1
@@ -80,6 +98,25 @@ function generateProvisionedSystemProperties() {
BACKEND_URLS="$BACKEND_URLS http://localhost:$MAPPED_PORT/auth"
done
echo "keycloak.backend.servers=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+
+ echo "# JMX" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_dc1_1
+ KC_DC1_PORT=$MAPPED_PORT
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_loadbalancer_dc2_1
+ KC_DC2_PORT=$MAPPED_PORT
+ echo "keycloak.frontend.servers.jmx=service:jmx:remote+http://localhost:$KC_DC1_PORT service:jmx:remote+http://localhost:$KC_DC2_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+ BACKEND_URLS=""
+ for SERVICE in $KEYCLOAK_SERVICES ; do
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_${SERVICE}_1
+ BACKEND_URLS="$BACKEND_URLS service:jmx:remote+http://localhost:$MAPPED_PORT"
+ done
+ echo "keycloak.backend.servers.jmx=$BACKEND_URLS" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
+
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_infinispan_dc1_1
+ ISPN_DC1_PORT=$MAPPED_PORT
+ inspectDockerPortMapping 9990/tcp ${PROJECT_NAME}_infinispan_dc2_1
+ ISPN_DC2_PORT=$MAPPED_PORT
+ echo "infinispan.servers.jmx=service:jmx:remote+http://localhost:$ISPN_DC1_PORT service:jmx:remote+http://localhost:$ISPN_DC2_PORT" >> $PROVISIONED_SYSTEM_PROPERTIES_FILE
;;
esac
}
diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml
index 818f8c95be..505db68c06 100644
--- a/testsuite/performance/tests/pom.xml
+++ b/testsuite/performance/tests/pom.xml
@@ -391,6 +391,9 @@
${project.version}
+ ${management.user}
+ ${management.user.password}
+
${keycloak.scale}
${keycloak.dc1.scale}
${keycloak.dc2.scale}
diff --git a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml
index 7d152c7d4b..4b8cb7c82d 100644
--- a/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml
+++ b/testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml
@@ -47,5 +47,6 @@ services:
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
ports:
- "8080:8080"
+ - "9990:9990"
diff --git a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml
index e51e192dcb..f92062eeb4 100644
--- a/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml
+++ b/testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml
@@ -35,6 +35,8 @@ services:
args:
LOCAL_SITE: dc1
REMOTE_SITE: dc2
+ MANAGEMENT_USER: ${MANAGEMENT_USER}
+ MANAGEMENT_USER_PASS: ${MANAGEMENT_USER_PASS}
image: keycloak_test_infinispan_dc1:${KEYCLOAK_VERSION:-latest}
cpuset: ${INFINISPAN_DC1_CPUSET:-1}
mem_limit: ${INFINISPAN_MEMLIMIT:-1500m}
@@ -49,7 +51,7 @@ services:
TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600]
JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
ports:
- - "9991:9990"
+ - "9990"
infinispan_dc2:
build:
@@ -57,6 +59,8 @@ services:
args:
LOCAL_SITE: dc2
REMOTE_SITE: dc1
+ MANAGEMENT_USER: ${MANAGEMENT_USER}
+ MANAGEMENT_USER_PASS: ${MANAGEMENT_USER_PASS}
image: keycloak_test_infinispan_dc2:${KEYCLOAK_VERSION:-latest}
depends_on:
infinispan_dc1:
@@ -74,7 +78,7 @@ services:
TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600]
JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
ports:
- - "9992:9990"
+ - "9990"
mariadb_dc1:
@@ -94,7 +98,7 @@ services:
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep-new-cluster
ports:
- - "3306:3306"
+ - "3307:3306"
mariadb_dc2:
build: db/mariadb
@@ -113,7 +117,7 @@ services:
entrypoint: docker-entrypoint-wsrep.sh
command: --wsrep_cluster_address=gcomm://mariadb_dc1
ports:
- - "3307:3306"
+ - "3308:3306"
loadbalancer_dc1:
@@ -133,6 +137,7 @@ services:
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
ports:
- "8081:8080"
+ - "9991:9990"
loadbalancer_dc2:
build: load-balancer/wildfly-modcluster
@@ -151,4 +156,5 @@ services:
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
ports:
- "8082:8080"
+ - "9992:9990"