KEYCLOAK-6288 Add support for "sar" metrics for docker-based performance testing

testsuite/performance/README.md

@@ -193,7 +193,14 @@ By default the monitoring history is preserved. If you wish to delete it enable
 
 To view monitoring dashboard open Grafana UI at: `http://localhost:3000/dashboard/file/resource-usage-combined.json`.
 
+### Sysstat metrics
 
+To enable recording of sysstat metrics use `-Psar`.
+This will run the `sar` command during the test and process its binary output to produce textual and CSV files with CPU utilisation stats.
+To also enable creation of PNG charts use `-Psar,gnuplot`. For this to work Gnuplot needs to be installed on the machine.
+To compress the binary output with bzip add `-Dbzip=true` to the commandline.
+
+Results will be stored in folder: `tests/target/sar`.
 
 ## Examples
 

testsuite/performance/tests/datasets/2ku200c.properties

@@ -0,0 +1,8 @@
+numOfRealms=1
+usersPerRealm=2000
+clientsPerRealm=200
+realmRoles=2
+realmRolesPerUser=2
+clientRolesPerUser=2
+clientRolesPerClient=2
+hashIterations=27500

testsuite/performance/tests/pom.xml

@@ -747,6 +747,58 @@
                 <delete.monitoring.data>true</delete.monitoring.data>
             </properties>
         </profile>
+        
+        <profile>
+            <id>sar</id>
+            <properties>
+                <gnuplot>false</gnuplot>
+                <bzip>false</bzip>
+            </properties>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>start-sar</id>
+                                <phase>pre-integration-test</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>./sar.sh</executable>
+                                    <environmentVariables>
+                                        <SAR_OPERATION>start</SAR_OPERATION>
+                                    </environmentVariables>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>stop-sar</id>
+                                <phase>post-integration-test</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>./sar.sh</executable>
+                                    <environmentVariables>
+                                        <SAR_OPERATION>stop</SAR_OPERATION>
+                                        <GNUPLOT>${gnuplot}</GNUPLOT>
+                                        <BZIP>${bzip}</BZIP>
+                                    </environmentVariables>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>gnuplot</id>
+            <properties>
+                <gnuplot>true</gnuplot>
+            </properties>
+        </profile>
 
     </profiles>
 

testsuite/performance/tests/sar.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+. ./common.sh
+
+SAR_OPERATION=${SAR_OPERATION:-stop}
+
+SAR_FOLDER="$PROJECT_BUILD_DIRECTORY/sar"
+PID_FILE="$SAR_FOLDER/sar.pid"
+TIMESTAMP_FILE="$SAR_FOLDER/sar.timestamp"
+if [[ -f "$TIMESTAMP_FILE" ]]; then 
+    TIMESTAMP=`cat $TIMESTAMP_FILE`
+else
+    TIMESTAMP=`date +%s`
+fi
+SAR_RESULTS_FOLDER="$SAR_FOLDER/$TIMESTAMP"
+SAR_OUTPUT_FILE="$SAR_RESULTS_FOLDER/sar-output.bin"
+BZIP=${BZIP:-false}
+CPU_COUNT=${CPU_COUNT:-`grep -c ^processor /proc/cpuinfo`}
+
+GNUPLOT=${GNUPLOT:-false}
+GNUPLOT_SCRIPTS_DIR="$PROJECT_BASEDIR/src/main/gnuplot/sar"
+GNUPLOT_COMMON="$GNUPLOT_SCRIPTS_DIR/common.gplot"
+
+function process_cpu_results() {
+    RESULTS_FOLDER="$SAR_RESULTS_FOLDER/cpu"; mkdir -p "$RESULTS_FOLDER"
+    CPU=${1:-ALL}
+    if [ "$CPU" == "ALL" ]; then SAR_PARAMS="-u"; else SAR_PARAMS="-u -P $CPU"; fi
+    TXT_FILE="$RESULTS_FOLDER/cpu-$CPU.txt"
+    CSV_FILE="${TXT_FILE%.txt}.csv"
+    PNG_FILE="${TXT_FILE%.txt}.png"
+    sar         $SAR_PARAMS -f   $SAR_OUTPUT_FILE > "$TXT_FILE"
+    sadf -d --  $SAR_PARAMS      $SAR_OUTPUT_FILE > "$CSV_FILE"
+    if $GNUPLOT; then
+        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cpu.gplot" > "$PNG_FILE"
+    fi
+}
+
+#function process_net_results() {
+#    IFACE=${IFACE:-docker0}
+#    RESULTS_FOLDER="$SAR_RESULTS_FOLDER/net"; mkdir -p "$RESULTS_FOLDER"
+#    TXT_FILE="$RESULTS_FOLDER/net-$IFACE.txt"
+#    CSV_FILE="${TXT_FILE%.txt}.csv"
+#    PNG_FILE="${TXT_FILE%.txt}.png"
+#    sar         -n DEV -f   $SAR_OUTPUT_FILE > "${TXT_FILE}.tmp"
+#    sadf -d --  -n DEV      $SAR_OUTPUT_FILE > "${CSV_FILE}.tmp"
+#    head -n 3 "${TXT_FILE}.tmp" > "$TXT_FILE"; grep "$IFACE" "${TXT_FILE}.tmp" >> "$TXT_FILE"; rm "${TXT_FILE}.tmp"
+#    head -n 1 "${CSV_FILE}.tmp" > "$CSV_FILE"; grep "$IFACE" "${CSV_FILE}.tmp" >> "$CSV_FILE"; rm "${CSV_FILE}.tmp"
+#    if $GNUPLOT; then
+#        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/net.gplot" > "$PNG_FILE"
+#    fi
+#}
+
+function process_io_results() {
+    RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
+    TXT_FILE="$RESULTS_FOLDER/io.txt"
+    CSV_FILE="${TXT_FILE%.txt}.csv"
+    sar         -b -f   $SAR_OUTPUT_FILE > "${TXT_FILE}"
+    sadf -d --  -b      $SAR_OUTPUT_FILE > "${CSV_FILE}"
+    if $GNUPLOT; then
+        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-requests.gplot" > "${TXT_FILE%.txt}-requests.png"
+        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-data.gplot" > "${TXT_FILE%.txt}-data.png"
+    fi
+}
+
+function process_mem_results() {
+    RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
+    TXT_FILE="$RESULTS_FOLDER/mem.txt"
+    CSV_FILE="${TXT_FILE%.txt}.csv"
+    PNG_FILE="${TXT_FILE%.txt}.png"
+    sar         -r -f   $SAR_OUTPUT_FILE > "${TXT_FILE}"
+    sadf -d --  -r      $SAR_OUTPUT_FILE > "${CSV_FILE}"
+    if $GNUPLOT; then
+        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/mem.gplot" > "$PNG_FILE"
+    fi
+}
+
+function process_cswch_results() {
+    RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
+    TXT_FILE="$RESULTS_FOLDER/cswch.txt"
+    CSV_FILE="${TXT_FILE%.txt}.csv"
+    PNG_FILE="${TXT_FILE%.txt}.png"
+    sar         -w -f   $SAR_OUTPUT_FILE > "${TXT_FILE}"
+    sadf -d --  -w      $SAR_OUTPUT_FILE > "${CSV_FILE}"
+    if $GNUPLOT; then
+        gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cswch.gplot" > "$PNG_FILE"
+    fi
+}
+
+
+case "$SAR_OPERATION" in
+
+    start)
+        if [[ ! -f "$PID_FILE" ]]; then
+            echo "Starting sar command."
+            mkdir -p $SAR_RESULTS_FOLDER
+            echo $TIMESTAMP > $TIMESTAMP_FILE
+            sar -A -o "$SAR_OUTPUT_FILE" 2 &>/dev/null & SAR_PID=$! && echo $SAR_PID > $PID_FILE
+        fi
+    ;;
+
+    stop)
+        if [[ -f "$PID_FILE" ]]; then
+            echo "Stopping sar command."
+            SAR_PID=`cat $PID_FILE`
+            kill $SAR_PID && rm $PID_FILE && rm $TIMESTAMP_FILE
+
+            echo "Processing sar output. GNUPLOT: $GNUPLOT"
+
+            # CPU
+            mkdir $SAR_RESULTS_FOLDER/cpu
+
+            process_cpu_results
+            for CPU in $(seq -f "%02g" 0 $(( CPU_COUNT-1 )) ); do 
+                process_cpu_results $CPU
+            done
+
+#            for IFACE in $(ls /sys/class/net); do 
+#                process_net_results $IFACE
+#            done
+
+            process_io_results 
+            process_mem_results 
+            process_cswch_results 
+
+            if $BZIP; then bzip2 "$SAR_OUTPUT_FILE"; fi
+
+            echo "Done."
+        fi
+    ;;
+
+esac

testsuite/performance/tests/src/main/gnuplot/sar/common.gplot

@@ -0,0 +1,11 @@
+set datafile separator ";"
+set datafile commentschar ""
+set xlabel "Time"
+set xdata time
+set timefmt "%Y-%m-%d %H:%M:%S"
+set format x '%H:%M:%S'
+set terminal pngcairo size 800,500
+set xtics rotate
+set yrange [0:*]
+set key below
+set grid

testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot

@@ -0,0 +1,11 @@
+set ylabel "% Utilization"
+set title "Processor Utilization"
+plot \
+for [i=5:8:1] \
+    datafile using 3:(sum [col=i:8] column(col)) \
+    title columnheader(i) \
+    with filledcurves x1, \
+for [i=5:8:1] \
+    datafile using 3:(sum [col=i:8] column(col)) \
+    notitle \
+    with lines lc rgb "#000000" lw 1

testsuite/performance/tests/src/main/gnuplot/sar/cswch.gplot

@@ -0,0 +1,2 @@
+set title "Context Switches Per Second"
+plot datafile using 3:5 title columnheader(5) with lines

testsuite/performance/tests/src/main/gnuplot/sar/io-data.gplot

@@ -0,0 +1,5 @@
+set title "IO Data - Blocks Read/Written Per Second"
+plot for [i=7:8:1] \
+    datafile using 3:i \
+    title columnheader(i) \
+    with lines

testsuite/performance/tests/src/main/gnuplot/sar/io-requests.gplot

@@ -0,0 +1,5 @@
+set title "IO Requests Per Second"
+plot for [i=4:6:1] \
+    datafile using 3:i \
+    title columnheader(i) \
+    with lines

testsuite/performance/tests/src/main/gnuplot/sar/mem.gplot

@@ -0,0 +1,8 @@
+set title "Memory Utilization"
+set ylabel "MB"
+plot \
+    datafile using 3:($5/1024) title 'Used' with filledcurves x1, \
+    datafile using 3:($5/1024) notitle with lines lc rgb "#000000" lw 1, \
+    datafile using 3:(($5 - $7 - $8)/1024) title 'Really Used' with filledcurves x1, \
+    datafile using 3:(($5 - $7 - $8)/1024) notitle with lines lc rgb "#000000" lw 1
+

testsuite/performance/tests/src/main/gnuplot/sar/net.gplot

@@ -0,0 +1,6 @@
+set ylabel "kB/S"
+set title "Network Usage"
+plot for [i=7:8:1] \
+    datafile using 3:i \
+    title columnheader(i) \
+    with lines