KEYCLOAK-6288 Add support for "sar" metrics for docker-based performance testing
This commit is contained in:
Tomas Kyjovsky
Hynek Mlnařík
parent
4ab856a0be
commit
3873ab811a
11 changed files with 247 additions and 0 deletions
|
|
@ -193,7 +193,14 @@ By default the monitoring history is preserved. If you wish to delete it enable
|
||||||
|
|
||||||
To view monitoring dashboard open Grafana UI at: `http://localhost:3000/dashboard/file/resource-usage-combined.json`.
|
To view monitoring dashboard open Grafana UI at: `http://localhost:3000/dashboard/file/resource-usage-combined.json`.
|
||||||
|
|
||||||
|
### Sysstat metrics
|
||||||
|
|
||||||
|
To enable recording of sysstat metrics use `-Psar`.
|
||||||
|
This will run the `sar` command during the test and process its binary output to produce textual and CSV files with CPU utilisation stats.
|
||||||
|
To also enable creation of PNG charts use `-Psar,gnuplot`. For this to work Gnuplot needs to be installed on the machine.
|
||||||
|
To compress the binary output with bzip add `-Dbzip=true` to the commandline.
|
||||||
|
|
||||||
|
Results will be stored in folder: `tests/target/sar`.
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
|
|
|
||||||
8
testsuite/performance/tests/datasets/2ku200c.properties
Normal file
8
testsuite/performance/tests/datasets/2ku200c.properties
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
numOfRealms=1
|
||||||
|
usersPerRealm=2000
|
||||||
|
clientsPerRealm=200
|
||||||
|
realmRoles=2
|
||||||
|
realmRolesPerUser=2
|
||||||
|
clientRolesPerUser=2
|
||||||
|
clientRolesPerClient=2
|
||||||
|
hashIterations=27500
|
||||||
|
|
@ -748,6 +748,58 @@
|
||||||
</properties>
|
</properties>
|
||||||
</profile>
|
</profile>
|
||||||
|
|
||||||
|
<profile>
|
||||||
|
<id>sar</id>
|
||||||
|
<properties>
|
||||||
|
<gnuplot>false</gnuplot>
|
||||||
|
<bzip>false</bzip>
|
||||||
|
</properties>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.codehaus.mojo</groupId>
|
||||||
|
<artifactId>exec-maven-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>start-sar</id>
|
||||||
|
<phase>pre-integration-test</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>./sar.sh</executable>
|
||||||
|
<environmentVariables>
|
||||||
|
<SAR_OPERATION>start</SAR_OPERATION>
|
||||||
|
</environmentVariables>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>stop-sar</id>
|
||||||
|
<phase>post-integration-test</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>exec</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<executable>./sar.sh</executable>
|
||||||
|
<environmentVariables>
|
||||||
|
<SAR_OPERATION>stop</SAR_OPERATION>
|
||||||
|
<GNUPLOT>${gnuplot}</GNUPLOT>
|
||||||
|
<BZIP>${bzip}</BZIP>
|
||||||
|
</environmentVariables>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
<profile>
|
||||||
|
<id>gnuplot</id>
|
||||||
|
<properties>
|
||||||
|
<gnuplot>true</gnuplot>
|
||||||
|
</properties>
|
||||||
|
</profile>
|
||||||
|
|
||||||
</profiles>
|
</profiles>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
132
testsuite/performance/tests/sar.sh
Executable file
132
testsuite/performance/tests/sar.sh
Executable file
|
|
@ -0,0 +1,132 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
. ./common.sh
|
||||||
|
|
||||||
|
SAR_OPERATION=${SAR_OPERATION:-stop}
|
||||||
|
|
||||||
|
SAR_FOLDER="$PROJECT_BUILD_DIRECTORY/sar"
|
||||||
|
PID_FILE="$SAR_FOLDER/sar.pid"
|
||||||
|
TIMESTAMP_FILE="$SAR_FOLDER/sar.timestamp"
|
||||||
|
if [[ -f "$TIMESTAMP_FILE" ]]; then
|
||||||
|
TIMESTAMP=`cat $TIMESTAMP_FILE`
|
||||||
|
else
|
||||||
|
TIMESTAMP=`date +%s`
|
||||||
|
fi
|
||||||
|
SAR_RESULTS_FOLDER="$SAR_FOLDER/$TIMESTAMP"
|
||||||
|
SAR_OUTPUT_FILE="$SAR_RESULTS_FOLDER/sar-output.bin"
|
||||||
|
BZIP=${BZIP:-false}
|
||||||
|
CPU_COUNT=${CPU_COUNT:-`grep -c ^processor /proc/cpuinfo`}
|
||||||
|
|
||||||
|
GNUPLOT=${GNUPLOT:-false}
|
||||||
|
GNUPLOT_SCRIPTS_DIR="$PROJECT_BASEDIR/src/main/gnuplot/sar"
|
||||||
|
GNUPLOT_COMMON="$GNUPLOT_SCRIPTS_DIR/common.gplot"
|
||||||
|
|
||||||
|
function process_cpu_results() {
|
||||||
|
RESULTS_FOLDER="$SAR_RESULTS_FOLDER/cpu"; mkdir -p "$RESULTS_FOLDER"
|
||||||
|
CPU=${1:-ALL}
|
||||||
|
if [ "$CPU" == "ALL" ]; then SAR_PARAMS="-u"; else SAR_PARAMS="-u -P $CPU"; fi
|
||||||
|
TXT_FILE="$RESULTS_FOLDER/cpu-$CPU.txt"
|
||||||
|
CSV_FILE="${TXT_FILE%.txt}.csv"
|
||||||
|
PNG_FILE="${TXT_FILE%.txt}.png"
|
||||||
|
sar $SAR_PARAMS -f $SAR_OUTPUT_FILE > "$TXT_FILE"
|
||||||
|
sadf -d -- $SAR_PARAMS $SAR_OUTPUT_FILE > "$CSV_FILE"
|
||||||
|
if $GNUPLOT; then
|
||||||
|
gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cpu.gplot" > "$PNG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
#function process_net_results() {
|
||||||
|
# IFACE=${IFACE:-docker0}
|
||||||
|
# RESULTS_FOLDER="$SAR_RESULTS_FOLDER/net"; mkdir -p "$RESULTS_FOLDER"
|
||||||
|
# TXT_FILE="$RESULTS_FOLDER/net-$IFACE.txt"
|
||||||
|
# CSV_FILE="${TXT_FILE%.txt}.csv"
|
||||||
|
# PNG_FILE="${TXT_FILE%.txt}.png"
|
||||||
|
# sar -n DEV -f $SAR_OUTPUT_FILE > "${TXT_FILE}.tmp"
|
||||||
|
# sadf -d -- -n DEV $SAR_OUTPUT_FILE > "${CSV_FILE}.tmp"
|
||||||
|
# head -n 3 "${TXT_FILE}.tmp" > "$TXT_FILE"; grep "$IFACE" "${TXT_FILE}.tmp" >> "$TXT_FILE"; rm "${TXT_FILE}.tmp"
|
||||||
|
# head -n 1 "${CSV_FILE}.tmp" > "$CSV_FILE"; grep "$IFACE" "${CSV_FILE}.tmp" >> "$CSV_FILE"; rm "${CSV_FILE}.tmp"
|
||||||
|
# if $GNUPLOT; then
|
||||||
|
# gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/net.gplot" > "$PNG_FILE"
|
||||||
|
# fi
|
||||||
|
#}
|
||||||
|
|
||||||
|
function process_io_results() {
|
||||||
|
RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
|
||||||
|
TXT_FILE="$RESULTS_FOLDER/io.txt"
|
||||||
|
CSV_FILE="${TXT_FILE%.txt}.csv"
|
||||||
|
sar -b -f $SAR_OUTPUT_FILE > "${TXT_FILE}"
|
||||||
|
sadf -d -- -b $SAR_OUTPUT_FILE > "${CSV_FILE}"
|
||||||
|
if $GNUPLOT; then
|
||||||
|
gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-requests.gplot" > "${TXT_FILE%.txt}-requests.png"
|
||||||
|
gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/io-data.gplot" > "${TXT_FILE%.txt}-data.png"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function process_mem_results() {
|
||||||
|
RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
|
||||||
|
TXT_FILE="$RESULTS_FOLDER/mem.txt"
|
||||||
|
CSV_FILE="${TXT_FILE%.txt}.csv"
|
||||||
|
PNG_FILE="${TXT_FILE%.txt}.png"
|
||||||
|
sar -r -f $SAR_OUTPUT_FILE > "${TXT_FILE}"
|
||||||
|
sadf -d -- -r $SAR_OUTPUT_FILE > "${CSV_FILE}"
|
||||||
|
if $GNUPLOT; then
|
||||||
|
gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/mem.gplot" > "$PNG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function process_cswch_results() {
|
||||||
|
RESULTS_FOLDER="$SAR_RESULTS_FOLDER"
|
||||||
|
TXT_FILE="$RESULTS_FOLDER/cswch.txt"
|
||||||
|
CSV_FILE="${TXT_FILE%.txt}.csv"
|
||||||
|
PNG_FILE="${TXT_FILE%.txt}.png"
|
||||||
|
sar -w -f $SAR_OUTPUT_FILE > "${TXT_FILE}"
|
||||||
|
sadf -d -- -w $SAR_OUTPUT_FILE > "${CSV_FILE}"
|
||||||
|
if $GNUPLOT; then
|
||||||
|
gnuplot -e "datafile='$CSV_FILE'" "$GNUPLOT_COMMON" "$GNUPLOT_SCRIPTS_DIR/cswch.gplot" > "$PNG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
case "$SAR_OPERATION" in
|
||||||
|
|
||||||
|
start)
|
||||||
|
if [[ ! -f "$PID_FILE" ]]; then
|
||||||
|
echo "Starting sar command."
|
||||||
|
mkdir -p $SAR_RESULTS_FOLDER
|
||||||
|
echo $TIMESTAMP > $TIMESTAMP_FILE
|
||||||
|
sar -A -o "$SAR_OUTPUT_FILE" 2 &>/dev/null & SAR_PID=$! && echo $SAR_PID > $PID_FILE
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
|
stop)
|
||||||
|
if [[ -f "$PID_FILE" ]]; then
|
||||||
|
echo "Stopping sar command."
|
||||||
|
SAR_PID=`cat $PID_FILE`
|
||||||
|
kill $SAR_PID && rm $PID_FILE && rm $TIMESTAMP_FILE
|
||||||
|
|
||||||
|
echo "Processing sar output. GNUPLOT: $GNUPLOT"
|
||||||
|
|
||||||
|
# CPU
|
||||||
|
mkdir $SAR_RESULTS_FOLDER/cpu
|
||||||
|
|
||||||
|
process_cpu_results
|
||||||
|
for CPU in $(seq -f "%02g" 0 $(( CPU_COUNT-1 )) ); do
|
||||||
|
process_cpu_results $CPU
|
||||||
|
done
|
||||||
|
|
||||||
|
# for IFACE in $(ls /sys/class/net); do
|
||||||
|
# process_net_results $IFACE
|
||||||
|
# done
|
||||||
|
|
||||||
|
process_io_results
|
||||||
|
process_mem_results
|
||||||
|
process_cswch_results
|
||||||
|
|
||||||
|
if $BZIP; then bzip2 "$SAR_OUTPUT_FILE"; fi
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
|
esac
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
set datafile separator ";"
|
||||||
|
set datafile commentschar ""
|
||||||
|
set xlabel "Time"
|
||||||
|
set xdata time
|
||||||
|
set timefmt "%Y-%m-%d %H:%M:%S"
|
||||||
|
set format x '%H:%M:%S'
|
||||||
|
set terminal pngcairo size 800,500
|
||||||
|
set xtics rotate
|
||||||
|
set yrange [0:*]
|
||||||
|
set key below
|
||||||
|
set grid
|
||||||
11
testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot
Normal file
11
testsuite/performance/tests/src/main/gnuplot/sar/cpu.gplot
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
set ylabel "% Utilization"
|
||||||
|
set title "Processor Utilization"
|
||||||
|
plot \
|
||||||
|
for [i=5:8:1] \
|
||||||
|
datafile using 3:(sum [col=i:8] column(col)) \
|
||||||
|
title columnheader(i) \
|
||||||
|
with filledcurves x1, \
|
||||||
|
for [i=5:8:1] \
|
||||||
|
datafile using 3:(sum [col=i:8] column(col)) \
|
||||||
|
notitle \
|
||||||
|
with lines lc rgb "#000000" lw 1
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
set title "Context Switches Per Second"
|
||||||
|
plot datafile using 3:5 title columnheader(5) with lines
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
set title "IO Data - Blocks Read/Written Per Second"
|
||||||
|
plot for [i=7:8:1] \
|
||||||
|
datafile using 3:i \
|
||||||
|
title columnheader(i) \
|
||||||
|
with lines
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
set title "IO Requests Per Second"
|
||||||
|
plot for [i=4:6:1] \
|
||||||
|
datafile using 3:i \
|
||||||
|
title columnheader(i) \
|
||||||
|
with lines
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
set title "Memory Utilization"
|
||||||
|
set ylabel "MB"
|
||||||
|
plot \
|
||||||
|
datafile using 3:($5/1024) title 'Used' with filledcurves x1, \
|
||||||
|
datafile using 3:($5/1024) notitle with lines lc rgb "#000000" lw 1, \
|
||||||
|
datafile using 3:(($5 - $7 - $8)/1024) title 'Really Used' with filledcurves x1, \
|
||||||
|
datafile using 3:(($5 - $7 - $8)/1024) notitle with lines lc rgb "#000000" lw 1
|
||||||
|
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
set ylabel "kB/S"
|
||||||
|
set title "Network Usage"
|
||||||
|
plot for [i=7:8:1] \
|
||||||
|
datafile using 3:i \
|
||||||
|
title columnheader(i) \
|
||||||
|
with lines
|
||||||
Loading…
Reference in a new issue