KEYCLOAK-8869 JStat monitoring for performance tests

testsuite/performance/README.md

@@ -302,6 +302,13 @@ To compress the binary output with bzip add `-Dbzip=true` to the commandline.
 
 Results will be stored in folder: `tests/target/sar`.
 
+### JStat - JVM memory statistics
+
+To enable jstat monitoring use `-Pjstat` option.
+This will start a `jstat` process in each container with Wildfly-based service (Keycloak, Infinispan, Load balancer)
+and record the statistics in the `standalone/log/jstat-gc.log` file. These can be then collected by running the `mvn verify -Pcollect` operation.
+
+To enable creation of PNG charts based on the jstat output use `-Pgnuplot`.
 
 ## Developing tests in IntelliJ IDEA
 

testsuite/performance/infinispan/infinispan.xml

@@ -67,6 +67,17 @@
         <delete dir="${infinispan.unpacked.home}/standalone/log"/>
         <delete dir="${infinispan.unpacked.home}/standalone/data"/>
         <delete dir="${infinispan.unpacked.home}/standalone/tmp"/>
+        <replace file="${infinispan.unpacked.home}/bin/standalone.sh">
+            <replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
+            <replacevalue><![CDATA[JBOSS_PID=$!
+
+      if [ "$JSTAT" = "true" ] ; then 
+        echo "Starting jstat"
+        mkdir -p $JBOSS_LOG_DIR
+        jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
+      fi
+]]></replacevalue>
+        </replace>
         <touch file="${infinispan.unpacked.home}/../configured"/>
     </target>
 

testsuite/performance/infinispan/src/main/scripts/Dockerfile

@@ -1,6 +1,7 @@
 FROM jboss/base-jdk:8
 
 ENV LAUNCH_JBOSS_IN_BACKGROUND 1
+ENV JSTAT false
 ENV CONFIGURATION clustered.xml
 ENV INFINISPAN_SERVER_HOME /opt/jboss/infinispan-server
 WORKDIR $INFINISPAN_SERVER_HOME

testsuite/performance/keycloak/configure.xml

@@ -45,6 +45,17 @@
         <delete dir="${server.unpacked.home}/standalone/log"/>
         <delete dir="${server.unpacked.home}/standalone/data"/>
         <delete dir="${server.unpacked.home}/standalone/tmp"/>
+        <replace file="${server.unpacked.home}/bin/standalone.sh">
+            <replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
+            <replacevalue><![CDATA[JBOSS_PID=$!
+
+      if [ "$JSTAT" = "true" ] ; then 
+        echo "Starting jstat"
+        mkdir -p $JBOSS_LOG_DIR
+        jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
+      fi
+]]></replacevalue>
+        </replace>
         <touch file="${project.build.directory}/performance-configured"/>
     </target>
 
@@ -73,7 +84,6 @@
         <touch file="${project.build.directory}/crossdc-configured"/>
     </target>
 
-
     <target name="keycloak-docker">
         <copy todir="${project.build.directory}/docker" overwrite="false">
             <fileset dir="${scripts.dir}">

testsuite/performance/keycloak/src/main/scripts/Dockerfile

@@ -9,6 +9,7 @@ ENV CONFIGURATION standalone.xml
 # ensuring clean shutdown when container is stopped.
 ENV LAUNCH_JBOSS_IN_BACKGROUND 1
 ENV PROXY_ADDRESS_FORWARDING false
+ENV JSTAT false
 
 USER root
 RUN yum install -y epel-release jq iproute && yum clean all

testsuite/performance/load-balancer/wildfly-modcluster/configure.xml

@@ -29,6 +29,17 @@
         <delete dir="${server.unpacked.home}/standalone/log"/>
         <delete dir="${server.unpacked.home}/standalone/data"/>
         <delete dir="${server.unpacked.home}/standalone/tmp"/>
+        <replace file="${server.unpacked.home}/bin/standalone.sh">
+            <replacetoken><![CDATA[JBOSS_PID=$!]]></replacetoken>
+            <replacevalue><![CDATA[JBOSS_PID=$!
+
+      if [ "$JSTAT" = "true" ] ; then 
+        echo "Starting jstat"
+        mkdir -p $JBOSS_LOG_DIR
+        jstat -gc -t $JBOSS_PID 1000 > $JBOSS_LOG_DIR/jstat-gc.log &
+      fi
+]]></replacevalue>
+        </replace>
         <touch file="${project.build.directory}/configured"/>
     </target>
 

testsuite/performance/load-balancer/wildfly-modcluster/src/main/scripts/Dockerfile

@@ -7,6 +7,7 @@ ENV CONFIGURATION standalone.xml
 
 # Ensure signals are forwarded to the JVM process correctly for graceful shutdown
 ENV LAUNCH_JBOSS_IN_BACKGROUND 1
+ENV JSTAT false
 
 USER root
 RUN yum -y install iproute

testsuite/performance/pom.xml

@@ -35,6 +35,10 @@
         <server.version>${product.version}</server.version>
         <management.user/>
         <management.user.password/>
+        <jstat>false</jstat>
+        <keycloak.jstat>${jstat}</keycloak.jstat>
+        <infinispan.jstat>${jstat}</infinispan.jstat>
+        <lb.jstat>${jstat}</lb.jstat>
     </properties>
             
     <modules>

testsuite/performance/tests/docker-compose.sh

@@ -417,14 +417,53 @@ case "$OPERATION" in
     ;;
 
     collect)
-        TIMESTAMP=`date +%s`
+        TIMESTAMP=`date -u "+%Y-%m-%d_%T_%Z"`
-        ARTIFACTS_DIR="${PROJECT_BUILD_DIRECTORY}/collected-artifacts/${DEPLOYMENT}-${TIMESTAMP}"
+        ARTIFACTS_DIR="${PROJECT_BUILD_DIRECTORY}/collected-artifacts/${DEPLOYMENT}_${TIMESTAMP}"
         SERVICES=`docker-compose -f $DOCKER_COMPOSE_FILE -p ${PROJECT_NAME} config --services`
-        echo "Collecting docker container logs."
+
+        GNUPLOT_SCRIPTS_DIR="$PROJECT_BASEDIR/src/main/gnuplot/jstat"
+        GNUPLOT_COMMON="$GNUPLOT_SCRIPTS_DIR/common.gp"
+
+        echo "Collecting service logs."
         rm -rf ${ARTIFACTS_DIR}; mkdir -p ${ARTIFACTS_DIR}
         for SERVICE in ${SERVICES}; do 
-            docker logs "${PROJECT_NAME}_${SERVICE}_1" > ${ARTIFACTS_DIR}/${SERVICE}.log 2>&1; 
+            mkdir -p "${ARTIFACTS_DIR}/${SERVICE}"
-            if [[ $? != 0 ]]; then echo "ERROR collecting from: ${SERVICE}"; rm ${ARTIFACTS_DIR}/${SERVICE}.log; fi
+            # log files & configs
+            if [[ $SERVICE =~ .*keycloak.* ]]; then 
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/keycloak/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/keycloak/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
+            elif [[ $SERVICE =~ .*infinispan.* ]]; then 
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/infinispan-server/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/infinispan-server/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
+            elif [[ $SERVICE =~ .*loadbalancer.* ]]; then 
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/wildfly/standalone/configuration" "${ARTIFACTS_DIR}/${SERVICE}/configuration"
+                docker cp "${PROJECT_NAME}_${SERVICE}_1:/opt/jboss/wildfly/standalone/log" "${ARTIFACTS_DIR}/${SERVICE}/log"
+            else
+                docker logs "${PROJECT_NAME}_${SERVICE}_1" > ${ARTIFACTS_DIR}/${SERVICE}/docker.log 2>&1; 
+                if [[ $? != 0 ]]; then echo "ERROR collecting from: ${SERVICE}"; rm ${ARTIFACTS_DIR}/${SERVICE}/docker.log; fi
+            fi
+
+#             jstat charts
+            if ${JSTAT:-false}; then
+                JSTAT_DATAFILE="${ARTIFACTS_DIR}/${SERVICE}/log/jstat-gc.log"
+                if [ -f "$JSTAT_DATAFILE" ] && ${GNUPLOT:-false}; then
+                    mkdir -p "${ARTIFACTS_DIR}/${SERVICE}/jstat-charts"
+                    HTML="${ARTIFACTS_DIR}/${SERVICE}/jstat-charts/index.html"
+                    echo "<html><head><title>JStat Charts for $SERVICE</title>" > "$HTML"
+                    echo "<style>div.box{ display: -webkit-inline-box }</style></head>" >> "$HTML"
+                    echo "<body><h1>JStat Charts for $SERVICE</h1>" >> "$HTML"
+                    for GP_SCRIPT in gc-all gc-s0 gc-s1 gc-e gc-o gc-m gc-cc ; do
+                        gnuplot -e "datafile='$JSTAT_DATAFILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/${GP_SCRIPT}.gp" > "${ARTIFACTS_DIR}/${SERVICE}/jstat-charts/${GP_SCRIPT}.png"
+                        if [ $? == 0 ]; then 
+                            echo "<div class='box'>" >> "$HTML"
+                            echo "<b>${GP_SCRIPT}</b><br/>" >> "$HTML"
+                            echo "<a href='${GP_SCRIPT}.png'><img src='${GP_SCRIPT}.png' width='400' height='300'/></a>" >> "$HTML"
+                            echo "</div>" >> "$HTML"
+                        fi
+                    done
+                    echo "</body></html>" >> "$HTML"
+                fi
+            fi
         done
         if [ -z "$(ls -A ${ARTIFACTS_DIR})" ]; then echo "No logs were collected."; rm -rf ${ARTIFACTS_DIR}; fi
     ;;

testsuite/performance/tests/pom.xml

@@ -431,6 +431,7 @@
                                         
                                         <MANAGEMENT_USER>${management.user}</MANAGEMENT_USER>
                                         <MANAGEMENT_USER_PASS>${management.user.password}</MANAGEMENT_USER_PASS>
+                                        <KEYCLOAK_JSTAT>${keycloak.jstat}</KEYCLOAK_JSTAT>
                                         
                                         <KEYCLOAK_SCALE>${keycloak.scale}</KEYCLOAK_SCALE>
                                         <KEYCLOAK_DC1_SCALE>${keycloak.dc1.scale}</KEYCLOAK_DC1_SCALE>
@@ -466,11 +467,13 @@
                                         <LB_HTTP_MAX_CONNECTIONS>${lb.http.max-connections}</LB_HTTP_MAX_CONNECTIONS>
                                         <LB_WORKER_IO_THREADS>${lb.worker.io-threads}</LB_WORKER_IO_THREADS>
                                         <LB_WORKER_TASK_MAX_THREADS>${lb.worker.task-max-threads}</LB_WORKER_TASK_MAX_THREADS>
+                                        <LB_JSTAT>${lb.jstat}</LB_JSTAT>
                                         
                                         <INFINISPAN_DC1_CPUSETS>${infinispan.dc1.docker.cpusets}</INFINISPAN_DC1_CPUSETS>
                                         <INFINISPAN_DC2_CPUSETS>${infinispan.dc2.docker.cpusets}</INFINISPAN_DC2_CPUSETS>
                                         <INFINISPAN_MEMLIMIT>${infinispan.docker.memlimit}</INFINISPAN_MEMLIMIT>
                                         <INFINISPAN_JVM_MEMORY>${infinispan.jvm.memory}</INFINISPAN_JVM_MEMORY>
+                                        <INFINISPAN_JSTAT>${infinispan.jstat}</INFINISPAN_JSTAT>
                                     </environmentVariables>
                                 </configuration>
                             </execution>
@@ -640,6 +643,9 @@
         
         <profile>
             <id>collect</id>
+            <properties>
+                <gnuplot>false</gnuplot>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -658,6 +664,8 @@
                                         <PROVISIONER>${provisioner}</PROVISIONER>
                                         <DEPLOYMENT>${deployment}</DEPLOYMENT>
                                         <OPERATION>collect</OPERATION>
+                                        <JSTAT>${jstat}</JSTAT>
+                                        <GNUPLOT>${gnuplot}</GNUPLOT>
                                     </environmentVariables>
                                 </configuration>
                             </execution>
@@ -826,6 +834,12 @@
                 <gnuplot>true</gnuplot>
             </properties>
         </profile>
+        <profile>
+            <id>jstat</id>
+            <properties>
+                <jstat>true</jstat>
+            </properties>
+        </profile>
 
     </profiles>
 

testsuite/performance/tests/prepare-provisioning.xml

@@ -10,10 +10,10 @@
                 <include name="monitoring/**"/>
             </fileset>
         </copy>
-        <copy todir="${project.build.directory}/docker-compose/infinispan" overwrite="false" failonerror="true">
+        <copy todir="${project.build.directory}/docker-compose/infinispan" overwrite="false" failonerror="false">
             <fileset dir="${project.basedir}/../infinispan/target/docker"/>
         </copy>
-        <copy todir="${project.build.directory}/docker-compose/load-balancer/wildfly-modcluster" overwrite="false" failonerror="true">
+        <copy todir="${project.build.directory}/docker-compose/load-balancer/wildfly-modcluster" overwrite="false" failonerror="false">
             <fileset dir="${project.basedir}/../load-balancer/wildfly-modcluster/target/docker"/>
         </copy>
         <copy todir="${project.build.directory}/docker-compose/keycloak" overwrite="false" failonerror="true">

testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-base.yml

@@ -48,6 +48,7 @@ services:
             HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
             WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
             WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
+            JSTAT: "${LB_JSTAT:-false}"
         ports:
             - "8080:8080"
             - "9990:9990"

testsuite/performance/tests/src/main/docker-compose/cluster/docker-compose-keycloak.yml

@@ -18,6 +18,7 @@
             MARIADB_PASSWORD: keycloak
             KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
             KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
+            JSTAT: "${KEYCLOAK_JSTAT:-false}"
 
             JAVA_OPTS: ${KEYCLOAK_JVM_MEMORY:--Xms64m -Xmx2g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
             HTTP_MAX_CONNECTIONS: ${KEYCLOAK_HTTP_MAX_CONNECTIONS:-50000}

testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-base.yml

@@ -45,6 +45,7 @@ services:
             MGMT_USER_PASSWORD: admin
             TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600]
             JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
+            JSTAT: "${INFINISPAN_JSTAT:-false}"
         ports:
             - "9990"
 
@@ -67,6 +68,7 @@ services:
             MGMT_USER_PASSWORD: admin
             TCP_PING_INITIAL_HOSTS: infinispan_dc1[7600],infinispan_dc2[7600]
             JAVA_OPTS: ${INFINISPAN_JVM_MEMORY:--Xms64m -Xmx1g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -XX:+DisableExplicitGC} -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true
+            JSTAT: "${INFINISPAN_JSTAT:-false}"
         ports:
             - "9990"
     
@@ -131,6 +133,7 @@ services:
             HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
             WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
             WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
+            JSTAT: "${LB_JSTAT:-false}"
         ports:
             - "8081:8080"
             - "9991:9990"
@@ -150,6 +153,7 @@ services:
             HTTP_MAX_CONNECTIONS: ${LB_HTTP_MAX_CONNECTIONS:-50000}
             WORKER_IO_THREADS: ${LB_WORKER_IO_THREADS:-2}
             WORKER_TASK_MAX_THREADS: ${LB_WORKER_TASK_MAX_THREADS:-16}
+            JSTAT: "${LB_JSTAT:-false}"
         ports:
             - "8082:8080"
             - "9992:9990"

testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc1.yml

@@ -22,6 +22,7 @@
             MARIADB_PASSWORD: keycloak
             KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
             KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
+            JSTAT: "${KEYCLOAK_JSTAT:-false}"
             INFINISPAN_HOST: infinispan_dc1
             SITE: dc1
             HOTROD_VERSION: 2.8

testsuite/performance/tests/src/main/docker-compose/crossdc/docker-compose-keycloak_dc2.yml

@@ -17,6 +17,9 @@
             MARIADB_DATABASE: keycloak
             MARIADB_USER: keycloak
             MARIADB_PASSWORD: keycloak
+            KEYCLOAK_ADMIN_USER: ${KEYCLOAK_ADMIN_USER:-admin}
+            KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin}
+            JSTAT: "${KEYCLOAK_JSTAT:-false}"
             INFINISPAN_HOST: infinispan_dc2
             SITE: dc2
             HOTROD_VERSION: 2.8

testsuite/performance/tests/src/main/docker-compose/docker-compose.yml

@@ -53,6 +53,7 @@ services:
             DS_MAX_POOL_SIZE: ${KEYCLOAK_DS_MAX_POOL_SIZE:-100}
             DS_POOL_PREFILL: "${KEYCLOAK_DS_POOL_PREFILL:-true}"
             DS_PS_CACHE_SIZE: ${KEYCLOAK_DS_PS_CACHE_SIZE:-100}
+            JSTAT: "${KEYCLOAK_JSTAT:-false}"
         ports:
             - "8080:8080"
             - "9990:9990"

testsuite/performance/tests/src/main/gnuplot/jstat/common.gp

@@ -0,0 +1,17 @@
+set datafile separator whitespace
+set datafile commentschar ""
+set xlabel "Runtime (s)"
+set ylabel "Memory (kB)"
+set terminal pngcairo size 1280,800
+set xtics rotate
+set yrange [0:*]
+set key below
+set grid
+
+set style fill solid 1.0 border -1
+set linetype 1 lc rgb '#ff0000'
+set linetype 2 lc rgb '#00ff39'
+set linetype 3 lc rgb '#4d00ff'
+set linetype 4 lc rgb '#ff00fb'
+set linetype 5 lc rgb '#00ffff'
+set linetype 6 lc rgb '#f7ff00'

testsuite/performance/tests/src/main/gnuplot/jstat/gc-all.gp

@@ -0,0 +1,8 @@
+set title "Heap + Non-heap Memory"
+plot\
+    datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')+column('MC')+column('CCSC')) title 'CCSC' with filledcurves x1, \
+    datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')+column('MC')) title 'MC' with filledcurves x1, \
+    datafile using 1:(column('S0C')+column('S1C')+column('EC')+column('OC')) title 'OC' with filledcurves x1, \
+    datafile using 1:(column('S0C')+column('S1C')+column('EC')) title 'EC' with filledcurves x1, \
+    datafile using 1:(column('S0C')+column('S1C')) title 'S1C' with filledcurves x1, \
+    datafile using 1:'S0C' title 'S0C' with filledcurves x1

testsuite/performance/tests/src/main/gnuplot/jstat/gc-cc.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Compressed Classes space"
+plot for [i in "CCSU CCSC"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/main/gnuplot/jstat/gc-e.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Eden space"
+plot for [i in "EU EC"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/main/gnuplot/jstat/gc-m.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Meta space"
+plot for [i in "MU MC"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/main/gnuplot/jstat/gc-o.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Old space"
+plot for [i in "OU OC"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/main/gnuplot/jstat/gc-s0.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Survivor 0 space"
+plot for [i in "S0U S0C"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/main/gnuplot/jstat/gc-s1.gp

@@ -0,0 +1,2 @@
+set title "Utilisation of Survivor 1 space"
+plot for [i in "S1U S1C"] datafile using 1:i title columnheader(i) with lines

testsuite/performance/tests/src/test/resources/dataset/1r_10c_100u_1hi.properties

@@ -0,0 +1,68 @@
+# REALM
+realms=1
+realm.realm=realm_${index}
+realm.displayName=Realm ${index}
+realm.enabled=true
+realm.registrationAllowed=true
+realm.accessTokenLifeSpan=60
+realm.passwordPolicy=hashIterations(1)
+
+# REALM ROLE
+realmRolesPerRealm=10
+realmRole.name=role_${index?string("00")}_of_${realm.realm}
+realmRole.description=Role ${index} of ${realm.displayName}
+
+# CLIENT
+clientsPerRealm=10
+client.clientId=client_${index?string("00")}_of_${realm.realm}
+client.name=Client ${index} of ${realm.displayName}
+client.description=Description of ${name}
+client.rootUrl=
+client.adminUrl=
+client.baseUrl=http://clients.${realm.realm}.test/client_${index}
+client.enabled=true
+client.secret=secret_of_${clientId}
+client.redirectUris=${baseUrl}/*
+client.webOrigins=
+client.protocol=openid-connect
+client.publicClient=<#if index % 3 == 0>true<#else>false</#if>
+client.bearerOnly=<#if index % 3 == 1>true<#else>false</#if>
+client.authorizationServicesEnabled=false
+client.serviceAccountsEnabled=${authorizationServicesEnabled?c}
+
+# CLIENT ROLE
+clientRolesPerClient=10
+clientRole.name=clientrole_${index?string("00")}_of_${client.clientId}
+clientRole.description=Role ${index} of ${client.name}
+
+# USER
+usersPerRealm=100
+user.username=user_${index?string("00")}_of_${realm.realm}
+user.enabled=true
+user.email=${username}@email.test
+user.emailVerified=true
+user.firstName=User_${index}
+user.lastName=O'Realm_${realm.index}
+
+credential.type=password
+credential.value=password_${index}_of_${user.username}
+credential.temporary=false
+
+# USER ATTRIBUTE
+attributesPerUser=3
+userAttribute.name=attribute_${index?string("00")}
+userAttribute.value=<#list 0..2 as i>value_${i}_of_${name}<#sep>,</#sep></#list>
+
+# USER ROLE MAPPINGS
+realmRolesPerUser=1
+clientRolesPerUser=3
+
+
+# GROUP
+groupsPerRealm=3
+group.name=group_${index?string("00")}_of ${realm.realm}
+
+# GROUP ATTRIBUTE
+attributesPerGroup=3
+groupAttribute.name=attribute_${index?string("00")}
+groupAttribute.value=<#list 0..2 as i>value_${i}_of_${name}<#sep>,</#sep></#list>