From af7e42d6402d5b5f8940eb15a4d31b46aa0bcb75 Mon Sep 17 00:00:00 2001 From: Tomas Kyjovsky Date: Mon, 26 Mar 2018 18:05:29 +0200 Subject: [PATCH] KEYCLOAK-5830 Automated stress test --- testsuite/performance/README.md | 5 + testsuite/performance/README.stress-test.md | 68 ++++++++++ testsuite/performance/stress-test-config.sh | 22 ++++ testsuite/performance/stress-test.sh | 117 ++++++++++++++++++ .../org/keycloak/performance/TestConfig.java | 11 ++ .../org/keycloak/performance/log/LogLine.java | 8 +- .../performance/log/LogProcessor.java | 5 + .../scala/keycloak/CommonSimulation.scala | 2 + .../OIDCLoginAndLogoutSimulation.scala | 7 +- .../OIDCRegisterAndLogoutSimulation.scala | 7 +- 10 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 testsuite/performance/README.stress-test.md create mode 100755 testsuite/performance/stress-test-config.sh create mode 100755 testsuite/performance/stress-test.sh diff --git a/testsuite/performance/README.md b/testsuite/performance/README.md index b06d4838aa..0fcd6ff683 100644 --- a/testsuite/performance/README.md +++ b/testsuite/performance/README.md @@ -200,6 +200,11 @@ When running the tests it is necessary to define the dataset to be used. | `userThinkTime` | Pause between individual scenario steps. | `5` | | `refreshTokenPeriod`| Period after which token should be refreshed. | `10` | +| Test Assertion | Description | Default Value | +| --- | --- | --- | +| `maxFailedRequests`| Maximum number of failed requests. | `0` | +| `maxMeanReponseTime`| Maximum mean response time of all requests. | `300` | + #### Test Run Parameters specific to `OIDCLoginAndLogoutSimulation` | Parameter | Description | Default Value | diff --git a/testsuite/performance/README.stress-test.md b/testsuite/performance/README.stress-test.md new file mode 100644 index 0000000000..978a983fec --- /dev/null +++ b/testsuite/performance/README.stress-test.md @@ -0,0 +1,68 @@ +# Stress Testing + +Stress testing is a type of performance testing focused on *finding the maximum performance* of the system for a specific scenario. + +There are various strategies but in general the stress test is a cycle of individual tests runs. +After each run the performance assertions are evaluated before deciding if/how the loop should continue. + +The [test assertions](https://gatling.io/docs/2.3/general/assertions/) are constructed as boolean expressions on top of computed performance metrics, such as mean response time, percentage of failed requests, etc. + + +## Requirements + +- `bc` tool for floating-point arithmetic + + +## Usage + +`./stress-test.sh [ADDITIONAL_TEST_PARAMS]` + +Parameters of the stress test are loaded from `stress-test-config.sh`. + +Additional `PROVISIONING_PARAMETERS` can be set via environment variable. + +## Common Parameters + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `algorithm` | Stress test loop algorithm. Available values: `incremental`, `bisection`. | `incremental` | +| `provisioning` | When `true` (enabled), the `provision` and `import-dump` operations are run before, and the `teardown` operation is run after test in each iteration. Warm-up is applied in all iterations. When `false` (disabled), there is no provisioning or teardown, and the warm-up is only applied in the first iteration. | `true` (enabled) | +| `PROVISIONING_PARAMETERS` | Additional set of parameters passed to the provisioning command. | | +| `maxIterations` | Maximum number of iterations of the stress test loop. | `10` iterations | +| `dataset` | Dataset to be used. | `100u2c` | +| `warmUpPeriod` | Sets value of `warmUpPeriod` parameter. If `provisioning` is disabled the warm-up is only done in the first iteration. | `120` seconds | +| `sequentialUsersFrom` | Value for the `sequentialUsersFrom` test parameter. If provisioning is disabled the value passed to the test command will be multiplied with each iteration. To be used with registration test scenario. | `-1` (random user iteration) | + + +## Incremental Method + +Incremental stress test is a loop with gradually increasing load being put on the system. +The cycle breaks with the first loop that fails the performance assertions, or after a maximum number of iterations + +It is useful for testing how various performance metrics evolve dependning on linear increments of load. + +### Parameters of Incremental Stress Test + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `usersPerSec0` | Value of `usersPerSec` parameter for the first iteration. | `5` user per second | +| `incrementFactor` | Factor of increment of `usersPerSec` with each subsequent iteration. The `usersPerSec` for iteration `i` (counted from 0) is computed as `usersPerSec0 + i * incrementFactor`. | `1` | + + +## Bisection Method + +This method (also called interval halving method) halves an interval defined by the lowest and highest expected value. +The test is performed with a load value from the middle of the specified interval and depending on the result either the lower or the upper half is used in the next iteration. +The cycle breaks when the interval gets smaller than a specified tolerance value, or after a maximum number of iterations. + +If set up properly the bisection algorithm is typically faster and more precise than the incremental method. +However it doesn't show metrics evolving with the linear progression of load. + +### Parameters of Bisection Stress Test + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `lowPoint` | The lower bound of the halved interval. Should be set to the lowest reasonably expected value of maximum performance. | `0` users per second | +| `highPoint` | The upper bound of the halved interval. | `10` users per second | +| `tolerance` | Indicates the precision of measurement. The stress test loop stops when the size of the halved interval is lower than this value. | `1` users per second | + diff --git a/testsuite/performance/stress-test-config.sh b/testsuite/performance/stress-test-config.sh new file mode 100755 index 0000000000..7d11a055b6 --- /dev/null +++ b/testsuite/performance/stress-test-config.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# common settings +export algorithm=incremental +export provisioning=false +export maxIterations=10 + +export dataset=100u2c +export warmUpPeriod=120 +export sequentialUsersFrom=-1 + +# incremental +export usersPerSec0=5 +export incrementFactor=1 + +# bisection +export lowPoint=0.000 +export highPoint=10.000 +export tolerance=1.000 + +# other +export debug=false diff --git a/testsuite/performance/stress-test.sh b/testsuite/performance/stress-test.sh new file mode 100755 index 0000000000..ca4f9af928 --- /dev/null +++ b/testsuite/performance/stress-test.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +BASEDIR=$(cd "$(dirname "$0")"; pwd) +cd $BASEDIR + +. ./stress-test-config.sh + +MVN=${MVN:-mvn} +PROVISIONING_PARAMETERS=${PROVISIONING_PARAMETERS:-} +PROVISION_COMMAND="$MVN verify -P provision,import-dump $PROVISIONING_PARAMETERS -Ddataset=$dataset" +TEARDOWN_COMMAND="$MVN verify -P teardown" + +function runCommand { + echo " $1" + echo + if ! $debug; then eval "$1"; fi +} + +function runTest { + + # use specified warmUpPeriod only in the first iteration, or if provisioning is enabled + if [[ $i == 0 || $provisioning == true ]]; then + warmUpParameter="-DwarmUpPeriod=$warmUpPeriod "; + else + warmUpParameter="-DwarmUpPeriod=0 "; + fi + if [[ $sequentialUsersFrom == -1 || $provisioning == true ]]; then + sequentialUsers=$sequentialUsersFrom + else + sequentialUsers=`echo "$sequentialUsersFrom * ( $i + 1 )" | bc` + fi + + TEST_COMMAND="$MVN verify -Ptest $@ -Ddataset=$dataset $warmUpParameter -DfilterResults=true -DsequentialUsersFrom=$sequentialUsers -DusersPerSec=$usersPerSec" + + echo "ITERATION: $(( i+1 )) / $maxIterations $ITERATION_INFO" + echo + + if $provisioning; then + runCommand "$PROVISION_COMMAND" + if [[ $? != 0 ]]; then + echo "Provisioning failed." + runCommand "$TEARDOWN_COMMAND" || break + break + fi + runCommand "$TEST_COMMAND" + export testResult=$? + runCommand "$TEARDOWN_COMMAND" || exit 1 + else + runCommand "$TEST_COMMAND" + export testResult=$? + fi + + [[ $testResult != 0 ]] && echo "Test exit code: $testResult" + +} + + + +echo "Starting ${algorithm} stress test" +echo + +usersPerSecTop=0 + +case "${algorithm}" in + + incremental) + + for (( i=0; i < $maxIterations; i++)); do + + usersPerSec=`echo "$usersPerSec0 + $i * $incrementFactor" | bc` + + runTest $@ + + if [[ $testResult == 0 ]]; then + usersPerSecTop=$usersPerSec + else + echo "INFO: Last iteration failed. Stopping the loop." + break + fi + + done + + ;; + + bisection) + + for (( i=0; i < $maxIterations; i++)); do + + intervalSize=`echo "$highPoint - $lowPoint" | bc` + usersPerSec=`echo "$lowPoint + $intervalSize * 0.5" | bc` + if [[ `echo "$intervalSize < $tolerance" | bc` == 1 ]]; then echo "INFO: intervalSize < tolerance. Stopping the loop."; break; fi + if [[ `echo "$intervalSize < 0" | bc` == 1 ]]; then echo "ERROR: Invalid state: lowPoint > highPoint. Stopping the loop."; exit 1; fi + ITERATION_INFO="L: $lowPoint H: $highPoint intervalSize: $intervalSize tolerance: $tolerance" + + runTest $@ + + if [[ $testResult == 0 ]]; then + usersPerSecTop=$usersPerSec + echo "INFO: Last iteration succeeded. Continuing with the upper half of the interval." + lowPoint=$usersPerSec + else + echo "INFO: Last iteration failed. Continuing with the lower half of the interval." + highPoint=$usersPerSec + fi + + done + + ;; + + *) + echo "Algorithm '${algorithm}' not supported." + exit 1 + ;; + +esac + +echo "Highest load with passing test: $usersPerSecTop users per second" diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java index 1aef0c9e08..cba6b72ccc 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java @@ -103,6 +103,10 @@ public class TestConfig { serverUrisList = Arrays.asList(serverUris.split(" ")); serverUrisIterator = new LoopingIterator<>(serverUrisList); } + + // assertion properties + public static final int maxFailedRequests = Integer.getInteger("maxFailedRequests", 0); + public static final int maxMeanReponseTime = Integer.getInteger("maxMeanReponseTime", 300); // Users iterators by realm private static final ConcurrentMap> usersIteratorMap = new ConcurrentHashMap<>(); @@ -172,6 +176,13 @@ public class TestConfig { hashIterations); } + public static String toStringAssertionProperties() { + return String.format(" maxFailedRequests: %s\n" + + " maxMeanReponseTime: %s", + maxFailedRequests, + maxMeanReponseTime); + } + public static Iterator sequentialUsersIterator(final String realm) { return new Iterator() { diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogLine.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogLine.java index 4fa33de04e..333b2be71a 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogLine.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogLine.java @@ -89,7 +89,9 @@ class LogLine { LogLine parse() { String[] cols = rawLine.split("\\t"); - if ("RUN".equals(cols[2])) { + if ("ASSERTION".equals(cols[0])) { + type = Type.ASSERTION; + } else if ("RUN".equals(cols[2])) { type = Type.RUN; simulationClass = cols[0]; simulationId = cols[1]; @@ -139,6 +141,9 @@ class LogLine { */ public String compose() { switch (type()) { + case ASSERTION: { + return rawLine; + } case RUN: { return simulationClass + "\t" + simulationId + "\t" + type.caption() + "\t" + start + "\t"+ description +"\t2.0\t"; } @@ -160,6 +165,7 @@ class LogLine { enum Type { + ASSERTION("ASSERTION"), RUN("RUN"), REQUEST("REQUEST\t"), USER_START("USER\tSTART"), diff --git a/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogProcessor.java b/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogProcessor.java index 55f6889b99..1eb299cfc9 100644 --- a/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogProcessor.java +++ b/testsuite/performance/tests/src/main/java/org/keycloak/performance/log/LogProcessor.java @@ -192,6 +192,11 @@ public class LogProcessor { LogLine line; while ((line = reader.readLine()) != null) { + if (line.type() == LogLine.Type.ASSERTION) { + output.println(line.rawLine()); + continue; + } + if (line.type() == LogLine.Type.RUN) { // adjust start time of simulation line.setStart(start); diff --git a/testsuite/performance/tests/src/test/scala/keycloak/CommonSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/CommonSimulation.scala index ded6c150a1..68115578c5 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/CommonSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/CommonSimulation.scala @@ -23,6 +23,8 @@ abstract class CommonSimulation extends Simulation { println() println("Using dataset properties:\n" + TestConfig.toStringDatasetProperties) println() + println("Using assertion properties:\n" + TestConfig.toStringAssertionProperties) + println() println("Timestamps: \n" + TestConfig.toStringTimestamps) println() diff --git a/testsuite/performance/tests/src/test/scala/keycloak/OIDCLoginAndLogoutSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/OIDCLoginAndLogoutSimulation.scala index e284897fce..7dfedf4a8d 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/OIDCLoginAndLogoutSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/OIDCLoginAndLogoutSimulation.scala @@ -17,5 +17,10 @@ class OIDCLoginAndLogoutSimulation extends CommonSimulation { val usersScenario = scenario("Logging-in Users").exec(loginAndLogoutScenario.chainBuilder) setUp(usersScenario.inject(defaultInjectionProfile).protocols(httpDefault)) - + + .assertions( + global.failedRequests.count.lessThan(TestConfig.maxFailedRequests + 1), + global.responseTime.mean.lessThan(TestConfig.maxMeanReponseTime) + ) + } diff --git a/testsuite/performance/tests/src/test/scala/keycloak/OIDCRegisterAndLogoutSimulation.scala b/testsuite/performance/tests/src/test/scala/keycloak/OIDCRegisterAndLogoutSimulation.scala index f050f46d7b..7c2eb79cc4 100644 --- a/testsuite/performance/tests/src/test/scala/keycloak/OIDCRegisterAndLogoutSimulation.scala +++ b/testsuite/performance/tests/src/test/scala/keycloak/OIDCRegisterAndLogoutSimulation.scala @@ -17,5 +17,10 @@ class OIDCRegisterAndLogoutSimulation extends CommonSimulation { val usersScenario = scenario("Registering Users").exec(registerAndLogoutScenario.chainBuilder) setUp(usersScenario.inject(defaultInjectionProfile).protocols(httpDefault)) - + + .assertions( + global.failedRequests.count.lessThan(TestConfig.maxFailedRequests + 1), + global.responseTime.mean.lessThan(TestConfig.maxMeanReponseTime) + ) + }