KEYCLOAK-8869 JStat monitoring for performance tests

This commit is contained in:
Tomas Kyjovsky 2018-11-15 04:48:46 +01:00 committed by Marek Posolda
parent 362faf3adb
commit 49a86865cf
26 changed files with 223 additions and 8 deletions

View file

@ -302,6 +302,13 @@ To compress the binary output with bzip add `-Dbzip=true` to the commandline.
Results will be stored in folder: `tests/target/sar`.
### JStat - JVM memory statistics
To enable jstat monitoring use `-Pjstat` option.
This will start a `jstat` process in each container with Wildfly-based service (Keycloak, Infinispan, Load balancer)
and record the statistics in the `standalone/log/jstat-gc.log` file. These can be then collected by running the `mvn verify -Pcollect` operation.
To enable creation of PNG charts based on the jstat output use `-Pgnuplot`.
## Developing tests in IntelliJ IDEA

View file

@ -67,6 +67,17 @@
<delete dir="${infinispan.unpacked.home}/standalone/log"/>
<delete dir="${infinispan.unpacked.home}/standalone/data"/>
<delete dir="${infinispan.unpacked.home}/standalone/tmp"/>
<replace file="${infinispan.unpacked.home}/bin/standalone.sh">
<replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
<replacevalue><![CDATA[JBOSS_PID=$!
if [ "$JSTAT" = "true" ] ; then
echo "Starting jstat"
mkdir -p $JBOSS_LOG_DIR
jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
fi
]]></replacevalue>
</replace>
<touch file="${infinispan.unpacked.home}/../configured"/>
</target>

View file

@ -1,6 +1,7 @@
FROM jboss/base-jdk:8
ENV LAUNCH_JBOSS_IN_BACKGROUND 1
ENV JSTAT false
ENV CONFIGURATION clustered.xml
ENV INFINISPAN_SERVER_HOME /opt/jboss/infinispan-server
WORKDIR $INFINISPAN_SERVER_HOME

View file

@ -45,6 +45,17 @@
<delete dir="${server.unpacked.home}/standalone/log"/>
<delete dir="${server.unpacked.home}/standalone/data"/>
<delete dir="${server.unpacked.home}/standalone/tmp"/>
<replace file="${server.unpacked.home}/bin/standalone.sh">
<replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
<replacevalue><![CDATA[JBOSS_PID=$!
if [ "$JSTAT" = "true" ] ; then
echo "Starting jstat"
mkdir -p $JBOSS_LOG_DIR
jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
fi
]]></replacevalue>
</replace>
<touch file="${project.build.directory}/performance-configured"/>
</target>
@ -73,7 +84,6 @@
<touch file="${project.build.directory}/crossdc-configured"/>
</target>
<target name="keycloak-docker">
<copy todir="${project.build.directory}/docker" overwrite="false">
<fileset dir="${scripts.dir}">

View file

@ -9,6 +9,7 @@ ENV CONFIGURATION standalone.xml
# ensuring clean shutdown when container is stopped.
ENV LAUNCH_JBOSS_IN_BACKGROUND 1
ENV PROXY_ADDRESS_FORWARDING false
ENV JSTAT false
USER root
RUN yum install -y epel-release jq iproute && yum clean all

View file

@ -29,6 +29,17 @@
<delete dir="${server.unpacked.home}/standalone/log"/>
<delete dir="${server.unpacked.home}/standalone/data"/>
<delete dir="${server.unpacked.home}/standalone/tmp"/>
<replace file="${server.unpacked.home}/bin/standalone.sh">
<replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
<replacevalue><![CDATA[JBOSS_PID=$!
if [ "$JSTAT" = "true" ] ; then
echo "Starting jstat"
mkdir -p $JBOSS_LOG_DIR
jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
fi
]]></replacevalue>
</replace>
<touch file="${project.build.directory}/configured"/>
</target>

View file

@ -7,6 +7,7 @@ ENV CONFIGURATION standalone.xml
# Ensure signals are forwarded to the JVM process correctly for graceful shutdown
ENV LAUNCH_JBOSS_IN_BACKGROUND 1
ENV JSTAT false
USER root
RUN yum -y install iproute

View file

@ -35,6 +35,10 @@
<server.version>${product.version}</server.version>
<management.user/>
<management.user.password/>
<jstat>false</jstat>
<keycloak.jstat>${jstat}</keycloak.jstat>
<infinispan.jstat>${jstat}</infinispan.jstat>
<lb.jstat>${jstat}</lb.jstat>
</properties>
<modules>

View file

@ -417,14 +417,53 @@ case "$OPERATION" in
;;
collect)
TIMESTAMP=`date +%s`
ARTIFACTS_DIR="${PROJECT_BUILD_DIRECTORY}/collected-artifacts/${DEPLOYMENT}-${TIMESTAMP}"
TIMESTAMP=`date -u "+%Y-%m-%d_%T_%Z"`
ARTIFACTS_DIR="${PROJECT_BUILD_DIRECTORY}/collected-artifacts/${DEPLOYMENT}_${TIMESTAMP}"
SERVICES=`docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} config --services`
echo "Collecting docker container logs."
GNUPLOT_SCRIPTS_DIR="$PROJECT_BASEDIR/src/main/gnuplot/jstat"
GNUPLOT_COMMON="$GNUPLOT_SCRIPTS_DIR/common.gp"
echo "Collecting service logs."
rm -rf ${ARTIFACTS_DIR}; mkdir -p ${ARTIFACTS_DIR}
for SERVICE in ${SERVICES}; do
docker logs "${PROJECT_NAME}_${SERVICE}_1" > ${ARTIFACTS_DIR}/${SERVICE}.log 2>&1;
if [[ $? != 0 ]]; then echo "ERROR collecting from: ${SERVICE}"; rm ${ARTIFACTS_DIR}/${SERVICE}.log; fi
mkdir -p "${ARTIFACTS_DIR}/${SERVICE}"
# log files & configs
if [[ $SERVICE =~ .*keycloak.* ]]; then
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/keycloak/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/keycloak/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
elif [[ $SERVICE =~ .*infinispan.* ]]; then
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/infinispan-server/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/infinispan-server/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
elif [[ $SERVICE =~ .*loadbalancer.* ]]; then
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/wildfly/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/wildfly/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
else
docker logs "${PROJECT_NAME}_${SERVICE}_1" > ${ARTIFACTS_DIR}/${SERVICE}/docker.log 2>&1;
if [[ $? != 0 ]]; then echo "ERROR collecting from: ${SERVICE}"; rm ${ARTIFACTS_DIR}/${SERVICE}/docker.log; fi
fi
# jstat charts
if ${JSTAT:-false}; then
JSTAT_DATAFILE="${ARTIFACTS_DIR}/${SERVICE}/log/jstat-gc.log"
if [ -f "$JSTAT_DATAFILE" ] && ${GNUPLOT:-false}; then
mkdir -p "${ARTIFACTS_DIR}/${SERVICE}/jstat-charts"
HTML="${ARTIFACTS_DIR}/${SERVICE}/jstat-charts/index.html"
echo "<html><head><title>JStat Charts for $SERVICE</title>" > "$HTML"
echo "<style>div.box{ display: -webkit-inline-box }</style></head>" >> "$HTML"
echo "<body><h1>JStat Charts for $SERVICE</h1>" >> "$HTML"
for GP_SCRIPT in gc-all gc-s0 gc-s1 gc-e gc-o gc-m gc-cc ; do
gnuplot -e "datafile='$JSTAT_DATAFILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/${GP_SCRIPT}.gp" > "${ARTIFACTS_DIR}/${SERVICE}/jstat-charts/${GP_SCRIPT}.png"
if [ $? == 0 ]; then
echo "<div class='box'>" >> "$HTML"
echo "<b>${GP_SCRIPT}</b><br/>" >> "$HTML"
echo "<a href='${GP_SCRIPT}.png'><img src='${GP_SCRIPT}.png' width='400' height='300'/></a>" >> "$HTML"
echo "</div>" >> "$HTML"
fi
done
echo "</body></html>" >> "$HTML"
fi
fi
done
if [ -z "$(ls -A ${ARTIFACTS_DIR})" ]; then echo "No logs were collected."; rm -rf ${ARTIFACTS_DIR}; fi
;;

View file

@ -431,6 +431,7 @@
<MANAGEMENT_USER>${management.user}</MANAGEMENT_USER>
<MANAGEMENT_USER_PASS>${management.user.password}</MANAGEMENT_USER_PASS>
<KEYCLOAK_JSTAT>${keycloak.jstat}</KEYCLOAK_JSTAT>
<KEYCLOAK_SCALE>${keycloak.scale}</KEYCLOAK_SCALE>
<KEYCLOAK_DC1_SCALE>${keycloak.dc1.scale}</KEYCLOAK_DC1_SCALE>
@ -466,11 +467,13 @@
<LB_HTTP_MAX_CONNECTIONS>${lb.http.max-connections}</LB_HTTP_MAX_CONNECTIONS>
<LB_WORKER_IO_THREADS>${lb.worker.io-threads}</LB_WORKER_IO_THREADS>
<LB_WORKER_TASK_MAX_THREADS>${lb.worker.task-max-threads}</LB_WORKER_TASK_MAX_THREADS>
<LB_JSTAT>${lb.jstat}</LB_JSTAT>
<INFINISPAN_DC1_CPUSETS>${infinispan.dc1.docker.cpusets}</INFINISPAN_DC1_CPUSETS>
<INFINISPAN_DC2_CPUSETS>${infinispan.dc2.docker.cpusets}</INFINISPAN_DC2_CPUSETS>
<INFINISPAN_MEMLIMIT>${infinispan.docker.memlimit}</INFINISPAN_MEMLIMIT>
<INFINISPAN_JVM_MEMORY>${infinispan.jvm.memory}</INFINISPAN_JVM_MEMORY>
<INFINISPAN_JSTAT>${infinispan.jstat}</INFINISPAN_JSTAT>
</environmentVariables>
</configuration>
</execution>
@ -640,6 +643,9 @@
<profile>
<id>collect</id>
<properties>
<gnuplot>false</gnuplot>
</properties>
<build>
<plugins>
<plugin>
@ -658,6 +664,8 @@
<PROVISIONER>${provisioner}</PROVISIONER>
<DEPLOYMENT>${deployment}</DEPLOYMENT>
<OPERATION>collect</OPERATION>
<JSTAT>${jstat}</JSTAT>
<GNUPLOT>${gnuplot}</GNUPLOT>
</environmentVariables>
</configuration>
</execution>
@ -826,6 +834,12 @@
<gnuplot>true</gnuplot>
</properties>
</profile>
<profile>
<id>jstat</id>
<properties>
<jstat>true</jstat>
</properties>
</profile>
</profiles>

View file

@ -10,10 +10,10 @@
<include name="monitoring/**"/>
</fileset>
</copy>
<copy todir="${project.build.directory}/docker-compose/infinispan" overwrite="false" failonerror="true">
<copy todir="${project.build.directory}/docker-compose/infinispan" overwrite="false" failonerror="false">
<fileset dir="${project.basedir}/../infinispan/target/docker"/>
</copy>
<copy todir="${project.build.directory}/docker-compose/load-balancer/wildfly-modcluster" overwrite="false" failonerror="true">
<copy todir="${project.build.directory}/docker-compose/load-balancer/wildfly-modcluster" overwrite="false" failonerror="false">
<fileset dir="${project.basedir}/../load-balancer/wildfly-modcluster/target/docker"/>
</copy>
<copy todir="${project.build.directory}/docker-compose/keycloak" overwrite="false" failonerror="true">

View file

@ -48,6 +48,7 @@ services:
HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
JSTAT: "${LB_JSTAT:-false}"
ports:
- "8080:8080"
- "9990:9990"

View file

@ -18,6 +18,7 @@
MARIADB_PASSWORD: keycloak
KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
JSTAT: "${KEYCLOAK_JSTAT:-false}"
JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000}

View file

@ -45,6 +45,7 @@ services:
MGMT_USER_PASSWORD: admin
TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600]
JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
JSTAT: "${INFINISPAN_JSTAT:-false}"
ports:
- "9990"
@ -67,6 +68,7 @@ services:
MGMT_USER_PASSWORD: admin
TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600]
JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
JSTAT: "${INFINISPAN_JSTAT:-false}"
ports:
- "9990"
@ -131,6 +133,7 @@ services:
HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
JSTAT: "${LB_JSTAT:-false}"
ports:
- "8081:8080"
- "9991:9990"
@ -150,6 +153,7 @@ services:
HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
JSTAT: "${LB_JSTAT:-false}"
ports:
- "8082:8080"
- "9992:9990"

View file

@ -22,6 +22,7 @@
MARIADB_PASSWORD: keycloak
KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
JSTAT: "${KEYCLOAK_JSTAT:-false}"
INFINISPAN_HOST: infinispan_dc1
SITE: dc1
HOTROD_VERSION: 2.8

View file

@ -17,6 +17,9 @@
MARIADB_DATABASE: keycloak
MARIADB_USER: keycloak
MARIADB_PASSWORD: keycloak
KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
JSTAT: "${KEYCLOAK_JSTAT:-false}"
INFINISPAN_HOST: infinispan_dc2
SITE: dc2
HOTROD_VERSION: 2.8

View file

@ -53,6 +53,7 @@ services:
DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
JSTAT: "${KEYCLOAK_JSTAT:-false}"
ports:
- "8080:8080"
- "9990:9990"

View file

@ -0,0 +1,17 @@
set datafile separator whitespace
set datafile commentschar ""
set xlabel "Runtime (s)"
set ylabel "Memory (kB)"
set terminal pngcairo size 1280,800
set xtics rotate
set yrange [0:*]
set key below
set grid
set style fill solid 1.0 border -1
set linetype 1 lc rgb '#ff0000'
set linetype 2 lc rgb '#00ff39'
set linetype 3 lc rgb '#4d00ff'
set linetype 4 lc rgb '#ff00fb'
set linetype 5 lc rgb '#00ffff'
set linetype 6 lc rgb '#f7ff00'

View file

@ -0,0 +1,8 @@
set title "Heap + Non-heap Memory"
plot\
datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')+column('MC')+column('CCSC')) title 'CCSC' with filledcurves x1, \
datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')+column('MC')) title 'MC' with filledcurves x1, \
datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')) title 'OC' with filledcurves x1, \
datafile using 1:(column('S0C')+column('S1C')+column('EC')) title 'EC' with filledcurves x1, \
datafile using 1:(column('S0C')+column('S1C')) title 'S1C' with filledcurves x1, \
datafile using 1:'S0C' title 'S0C' with filledcurves x1

View file

@ -0,0 +1,2 @@
set title "Utilisation of Compressed Classes space"
plot for [i in "CCSU CCSC"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,2 @@
set title "Utilisation of Eden space"
plot for [i in "EU EC"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,2 @@
set title "Utilisation of Meta space"
plot for [i in "MU MC"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,2 @@
set title "Utilisation of Old space"
plot for [i in "OU OC"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,2 @@
set title "Utilisation of Survivor 0 space"
plot for [i in "S0U S0C"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,2 @@
set title "Utilisation of Survivor 1 space"
plot for [i in "S1U S1C"] datafile using 1:i title columnheader(i) with lines

View file

@ -0,0 +1,68 @@
# REALM
realms=1
realm.realm=realm_${index}
realm.displayName=Realm ${index}
realm.enabled=true
realm.registrationAllowed=true
realm.accessTokenLifeSpan=60
realm.passwordPolicy=hashIterations(1)
# REALM ROLE
realmRolesPerRealm=10
realmRole.name=role_${index?string("00")}_of_${realm.realm}
realmRole.description=Role ${index} of ${realm.displayName}
# CLIENT
clientsPerRealm=10
client.clientId=client_${index?string("00")}_of_${realm.realm}
client.name=Client ${index} of ${realm.displayName}
client.description=Description of ${name}
client.rootUrl=
client.adminUrl=
client.baseUrl=http://clients.${realm.realm}.test/client_${index}
client.enabled=true
client.secret=secret_of_${clientId}
client.redirectUris=${baseUrl}/*
client.webOrigins=
client.protocol=openid-connect
client.publicClient=<#if index % 3 == 0>true<#else>false</#if>
client.bearerOnly=<#if index % 3 == 1>true<#else>false</#if>
client.authorizationServicesEnabled=false
client.serviceAccountsEnabled=${authorizationServicesEnabled?c}
# CLIENT ROLE
clientRolesPerClient=10
clientRole.name=clientrole_${index?string("00")}_of_${client.clientId}
clientRole.description=Role ${index} of ${client.name}
# USER
usersPerRealm=100
user.username=user_${index?string("00")}_of_${realm.realm}
user.enabled=true
user.email=${username}@email.test
user.emailVerified=true
user.firstName=User_${index}
user.lastName=O'Realm_${realm.index}
credential.type=password
credential.value=password_${index}_of_${user.username}
credential.temporary=false
# USER ATTRIBUTE
attributesPerUser=3
userAttribute.name=attribute_${index?string("00")}
userAttribute.value=<#list 0..2 as i>value_${i}_of_${name}<#sep>,</#sep></#list>
# USER ROLE MAPPINGS
realmRolesPerUser=1
clientRolesPerUser=3
# GROUP
groupsPerRealm=3
group.name=group_${index?string("00")}_of ${realm.realm}
# GROUP ATTRIBUTE
attributesPerGroup=3
groupAttribute.name=attribute_${index?string("00")}
groupAttribute.value=<#list 0..2 as i>value_${i}_of_${name}<#sep>,</#sep></#list>