From 3873ab811a7137c5229e90956b0565141c6c4be8 Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Tue, 16 Jan 2018 23:13:38 +0100 Subject: [PATCH] KEYCLOAK-6288 Add support for "sar" metrics for docker-based performance testing --- testsuite/performance/README.md | 7 + .../tests/datasets/2ku200c.properties | 8 ++ testsuite/performance/tests/pom.xml | 52 +++++++ testsuite/performance/tests/sar.sh | 132 ++++++++++++++++++ .../tests/src/main/gnuplot/sar/common.gplot | 11 ++ .../tests/src/main/gnuplot/sar/cpu.gplot | 11 ++ .../tests/src/main/gnuplot/sar/cswch.gplot | 2 + .../tests/src/main/gnuplot/sar/io-data.gplot | 5 + .../src/main/gnuplot/sar/io-requests.gplot | 5 + .../tests/src/main/gnuplot/sar/mem.gplot | 8 ++ .../tests/src/main/gnuplot/sar/net.gplot | 6 + 11 files changed, 247 insertions(+) create mode 100644 testsuite/performance/tests/datasets/2ku200c.properties create mode 100755 testsuite/performance/tests/sar.sh create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/common.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/cswch.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/io-data.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/io-requests.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/mem.gplot create mode 100644 testsuite/performance/tests/src/main/gnuplot/sar/net.gplot diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index cccedd54ef..52e58b17c4 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -193,7 +193,14 @@ By default the monitoring history is preserved. If you wish to delete it enable To view monitoring dashboard open Grafana UI at: `http://localhost:3000/dashboard/file/resource-usage-combined.json`. +### Sysstat metrics +To enable recording of sysstat metrics use `-Psar`. +This will run the `sar` command during the test and process its binary output to produce textual and CSV files with CPU utilisation stats. +To also enable creation of PNG charts use `-Psar,gnuplot`. For this to work Gnuplot needs to be installed on the machine. +To compress the binary output with bzip add `-Dbzip=true` to the commandline. + +Results will be stored in folder: `tests/target/sar`. ## Examples diff --git a/testsuite/performance/tests/datasets/2ku200c.properties b/testsuite/performance/tests/datasets/2ku200c.properties new file mode 100644 index 0000000000..7b1591106b --- /dev/null +++ b/testsuite/performance/tests/datasets/2ku200c.properties @@ -0,0 +1,8 @@ +numOfRealms=1 +usersPerRealm=2000 +clientsPerRealm=200 +realmRoles=2 +realmRolesPerUser=2 +clientRolesPerUser=2 +clientRolesPerClient=2 +hashIterations=27500 diff --git a/testsuite/performance/tests/pom.xml b/testsuite/performance/tests/pom.xml index f13a1ac620..7173a172a9 100644 --- a/testsuite/performance/tests/pom.xml +++ b/testsuite/performance/tests/pom.xml @@ -747,6 +747,58 @@ true + + + sar + + false + false + + + + + org.codehaus.mojo + exec-maven-plugin + + + start-sar + pre-integration-test + + exec + + + ./sar.sh + + start + + + + + stop-sar + post-integration-test + + exec + + + ./sar.sh + + stop + ${gnuplot} + ${bzip} + + + + + + + + + + gnuplot + + true + + diff --git a/testsuite/performance/tests/sar.sh b/testsuite/performance/tests/sar.sh new file mode 100755 index 0000000000..d95f3ac22d --- /dev/null +++ b/testsuite/performance/tests/sar.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +cd "$(dirname "$0")" +. ./common.sh + +SAR_OPERATION=${SAR_OPERATION:-stop} + +SAR_FOLDER="$PROJECT_BUILD_DIRECTORY/sar" +PID_FILE="$SAR_FOLDER/sar.pid" +TIMESTAMP_FILE="$SAR_FOLDER/sar.timestamp" +if [[ -f "$TIMESTAMP_FILE" ]]; then + TIMESTAMP=`cat $TIMESTAMP_FILE` +else + TIMESTAMP=`date +%s` +fi +SAR_RESULTS_FOLDER="$SAR_FOLDER/$TIMESTAMP" +SAR_OUTPUT_FILE="$SAR_RESULTS_FOLDER/sar-output.bin" +BZIP=${BZIP:-false} +CPU_COUNT=${CPU_COUNT:-`grep -c ^processor /proc/cpuinfo`} + +GNUPLOT=${GNUPLOT:-false} +GNUPLOT_SCRIPTS_DIR="$PROJECT_BASEDIR/src/main/gnuplot/sar" +GNUPLOT_COMMON="$GNUPLOT_SCRIPTS_DIR/common.gplot" + +function process_cpu_results() { + RESULTS_FOLDER="$SAR_RESULTS_FOLDER/cpu"; mkdir -p "$RESULTS_FOLDER" + CPU=${1:-ALL} + if [ "$CPU" == "ALL" ]; then SAR_PARAMS="-u"; else SAR_PARAMS="-u -P $CPU"; fi + TXT_FILE="$RESULTS_FOLDER/cpu-$CPU.txt" + CSV_FILE="${TXT_FILE%.txt}.csv" + PNG_FILE="${TXT_FILE%.txt}.png" + sar $SAR_PARAMS -f $SAR_OUTPUT_FILE > "$TXT_FILE" + sadf -d -- $SAR_PARAMS $SAR_OUTPUT_FILE > "$CSV_FILE" + if $GNUPLOT; then + gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cpu.gplot" > "$PNG_FILE" + fi +} + +#function process_net_results() { +# IFACE=${IFACE:-docker0} +# RESULTS_FOLDER="$SAR_RESULTS_FOLDER/net"; mkdir -p "$RESULTS_FOLDER" +# TXT_FILE="$RESULTS_FOLDER/net-$IFACE.txt" +# CSV_FILE="${TXT_FILE%.txt}.csv" +# PNG_FILE="${TXT_FILE%.txt}.png" +# sar -n DEV -f $SAR_OUTPUT_FILE > "${TXT_FILE}.tmp" +# sadf -d -- -n DEV $SAR_OUTPUT_FILE > "${CSV_FILE}.tmp" +# head -n 3 "${TXT_FILE}.tmp" > "$TXT_FILE"; grep "$IFACE" "${TXT_FILE}.tmp" >> "$TXT_FILE"; rm "${TXT_FILE}.tmp" +# head -n 1 "${CSV_FILE}.tmp" > "$CSV_FILE"; grep "$IFACE" "${CSV_FILE}.tmp" >> "$CSV_FILE"; rm "${CSV_FILE}.tmp" +# if $GNUPLOT; then +# gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/net.gplot" > "$PNG_FILE" +# fi +#} + +function process_io_results() { + RESULTS_FOLDER="$SAR_RESULTS_FOLDER" + TXT_FILE="$RESULTS_FOLDER/io.txt" + CSV_FILE="${TXT_FILE%.txt}.csv" + sar -b -f $SAR_OUTPUT_FILE > "${TXT_FILE}" + sadf -d -- -b $SAR_OUTPUT_FILE > "${CSV_FILE}" + if $GNUPLOT; then + gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-requests.gplot" > "${TXT_FILE%.txt}-requests.png" + gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-data.gplot" > "${TXT_FILE%.txt}-data.png" + fi +} + +function process_mem_results() { + RESULTS_FOLDER="$SAR_RESULTS_FOLDER" + TXT_FILE="$RESULTS_FOLDER/mem.txt" + CSV_FILE="${TXT_FILE%.txt}.csv" + PNG_FILE="${TXT_FILE%.txt}.png" + sar -r -f $SAR_OUTPUT_FILE > "${TXT_FILE}" + sadf -d -- -r $SAR_OUTPUT_FILE > "${CSV_FILE}" + if $GNUPLOT; then + gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/mem.gplot" > "$PNG_FILE" + fi +} + +function process_cswch_results() { + RESULTS_FOLDER="$SAR_RESULTS_FOLDER" + TXT_FILE="$RESULTS_FOLDER/cswch.txt" + CSV_FILE="${TXT_FILE%.txt}.csv" + PNG_FILE="${TXT_FILE%.txt}.png" + sar -w -f $SAR_OUTPUT_FILE > "${TXT_FILE}" + sadf -d -- -w $SAR_OUTPUT_FILE > "${CSV_FILE}" + if $GNUPLOT; then + gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cswch.gplot" > "$PNG_FILE" + fi +} + + +case "$SAR_OPERATION" in + + start) + if [[ ! -f "$PID_FILE" ]]; then + echo "Starting sar command." + mkdir -p $SAR_RESULTS_FOLDER + echo $TIMESTAMP > $TIMESTAMP_FILE + sar -A -o "$SAR_OUTPUT_FILE" 2 &>/dev/null & SAR_PID=$! && echo $SAR_PID > $PID_FILE + fi + ;; + + stop) + if [[ -f "$PID_FILE" ]]; then + echo "Stopping sar command." + SAR_PID=`cat $PID_FILE` + kill $SAR_PID && rm $PID_FILE && rm $TIMESTAMP_FILE + + echo "Processing sar output. GNUPLOT: $GNUPLOT" + + # CPU + mkdir $SAR_RESULTS_FOLDER/cpu + + process_cpu_results + for CPU in $(seq -f "%02g" 0 $(( CPU_COUNT-1 )) ); do + process_cpu_results $CPU + done + +# for IFACE in $(ls /sys/class/net); do +# process_net_results $IFACE +# done + + process_io_results + process_mem_results + process_cswch_results + + if $BZIP; then bzip2 "$SAR_OUTPUT_FILE"; fi + + echo "Done." + fi + ;; + +esac diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/common.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/common.gplot new file mode 100644 index 0000000000..8bd59a8eae --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/common.gplot @@ -0,0 +1,11 @@ +set datafile separator ";" +set datafile commentschar "" +set xlabel "Time" +set xdata time +set timefmt "%Y-%m-%d %H:%M:%S" +set format x '%H:%M:%S' +set terminal pngcairo size 800,500 +set xtics rotate +set yrange [0:*] +set key below +set grid diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot new file mode 100644 index 0000000000..8bd00f9cdd --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot @@ -0,0 +1,11 @@ +set ylabel "% Utilization" +set title "Processor Utilization" +plot \ +for [i=5:8:1] \ + datafile using 3:(sum [col=i:8] column(col)) \ + title columnheader(i) \ + with filledcurves x1, \ +for [i=5:8:1] \ + datafile using 3:(sum [col=i:8] column(col)) \ + notitle \ + with lines lc rgb "#000000" lw 1 diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/cswch.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/cswch.gplot new file mode 100644 index 0000000000..caf1e3348d --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/cswch.gplot @@ -0,0 +1,2 @@ +set title "Context Switches Per Second" +plot datafile using 3:5 title columnheader(5) with lines diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/io-data.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/io-data.gplot new file mode 100644 index 0000000000..adaf658d74 --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/io-data.gplot @@ -0,0 +1,5 @@ +set title "IO Data - Blocks Read/Written Per Second" +plot for [i=7:8:1] \ + datafile using 3:i \ + title columnheader(i) \ + with lines diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/io-requests.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/io-requests.gplot new file mode 100644 index 0000000000..30c7de5d1b --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/io-requests.gplot @@ -0,0 +1,5 @@ +set title "IO Requests Per Second" +plot for [i=4:6:1] \ + datafile using 3:i \ + title columnheader(i) \ + with lines diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/mem.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/mem.gplot new file mode 100644 index 0000000000..500b59a876 --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/mem.gplot @@ -0,0 +1,8 @@ +set title "Memory Utilization" +set ylabel "MB" +plot \ + datafile using 3:($5/1024) title 'Used' with filledcurves x1, \ + datafile using 3:($5/1024) notitle with lines lc rgb "#000000" lw 1, \ + datafile using 3:(($5 - $7 - $8)/1024) title 'Really Used' with filledcurves x1, \ + datafile using 3:(($5 - $7 - $8)/1024) notitle with lines lc rgb "#000000" lw 1 + diff --git a/testsuite/performance/tests/src/main/gnuplot/sar/net.gplot b/testsuite/performance/tests/src/main/gnuplot/sar/net.gplot new file mode 100644 index 0000000000..44dd7e2d5c --- /dev/null +++ b/testsuite/performance/tests/src/main/gnuplot/sar/net.gplot @@ -0,0 +1,6 @@ +set ylabel "kB/S" +set title "Network Usage" +plot for [i=7:8:1] \ + datafile using 3:i \ + title columnheader(i) \ + with lines