libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases Packages Activity Actions

Updates to the performance tests.

Browse source 
Conflicts:
	testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml
	testsuite/integration-arquillian/tests/other/clean-start/pom.xml
This commit is contained in:
Tomas Kyjovsky 2016-05-04 02:48:36 +02:00
parent 2343e517c9
commit ef95510da4
34 changed files with 815 additions and 536 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json Normal file
View file

@ -0,0 +1,15 @@
[ {
"realm" : "master",
"users" : [ {
"username" : "admin",
"enabled" : true,
"credentials" : [ {
"type" : "password",
"hashedSaltedValue" : "dqalJHLkWhUJZO/q6+z1fvXOohTcGCXcvoU8xCEyvTxGN4wmLx7DtyhKuefggh6Bkx1I2eBTEX4tiWggwyXMDw==",
"salt" : "3fBAt5GAGGxFrV9fznpZHQ==",
"hashIterations" : 100000,
"algorithm" : "pbkdf2"
} ],
"realmRoles" : [ "admin" ]
} ]
} ]

20
testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl
View file

@ -23,7 +23,25 @@
<xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
<xsl:strip-space elements="*"/>
<!--add socket binding-->
<xsl:param name="load.metric" select="'simple'" />\
<!-- mod-cluster-config -->
<xsl:template match="//*[local-name()='mod-cluster-config']">
<mod-cluster-config advertise-socket="modcluster" connector="ajp">
<xsl:choose>
<xsl:when test="$load.metric='simple'">
<simple-load-provider factor="1"/>
</xsl:when>
<xsl:otherwise>
<dynamic-load-provider>
<load-metric type="{$load.metric}"/>
</dynamic-load-provider>
</xsl:otherwise>
</xsl:choose>
</mod-cluster-config>
</xsl:template>
<!--add socket-binding-->
<xsl:template match="//*[local-name()='socket-binding-group' and @name='standard-sockets']/*[local-name()='socket-binding' and @name='modcluster']">
<socket-binding name="modcluster" interface="private" port="0" multicast-address="${{jboss.default.multicast.address:230.0.0.4}}" multicast-port="23364"/>
</xsl:template>

154
testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml
View file

@ -406,60 +406,60 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>inject-truststore-into-keycloak-server-json</id>
<phase>process-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ant antfile="../build-truststore.xml" inheritRefs="true">
<target name="inject-truststore"/>
</ant>
</target>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>ant-contrib</groupId>
<artifactId>ant-contrib</artifactId>
<version>1.0b3</version>
<exclusions>
<exclusion>
<groupId>ant</groupId>
<artifactId>ant</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>
<artifactId>ant-apache-bsf</artifactId>
<version>1.9.3</version>
</dependency>
<dependency>
<groupId>org.apache.bsf</groupId>
<artifactId>bsf-api</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>rhino</groupId>
<artifactId>js</artifactId>
<version>1.7R2</version>
</dependency>
<dependency>
<groupId>org.keycloak</groupId>
<artifactId>keycloak-core</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>inject-truststore-into-keycloak-server-json</id>
<phase>process-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ant antfile="../build-truststore.xml" inheritRefs="true">
<target name="inject-truststore"/>
</ant>
</target>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>ant-contrib</groupId>
<artifactId>ant-contrib</artifactId>
<version>1.0b3</version>
<exclusions>
<exclusion>
<groupId>ant</groupId>
<artifactId>ant</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>
<artifactId>ant-apache-bsf</artifactId>
<version>1.9.3</version>
</dependency>
<dependency>
<groupId>org.apache.bsf</groupId>
<artifactId>bsf-api</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>rhino</groupId>
<artifactId>js</artifactId>
<version>1.7R2</version>
</dependency>
<dependency>
<groupId>org.keycloak</groupId>
<artifactId>keycloak-core</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</pluginManagement>
</build>
@ -615,6 +615,12 @@
<session.cache.owners>1</session.cache.owners>
<offline.session.cache.owners>1</offline.session.cache.owners>
<login.failure.cache.owners>1</login.failure.cache.owners>
<load.metric>simple</load.metric>
<!-- The default value 'simple' configures mod-cluster with simple-load-provider.
Any other value configures it with dynamic-load-provider using the particular `load.metric`.
Supported metrics: https://docs.jboss.org/mod_cluster/1.2.0/html/java.AS7config.html#LoadMetric -->
</properties>
<build>
<pluginManagement>
@ -701,6 +707,12 @@
</includes>
<stylesheet>${common.resources}/mod_cluster.xsl</stylesheet>
<outputDir>${auth.server.home}/standalone/configuration</outputDir>
<parameters>
<parameter>
<name>load.metric</name>
<value>${load.metric}</value>
</parameter>
</parameters>
</transformationSet>
</transformationSets>
</configuration>
@ -712,6 +724,40 @@
</build>
</profile>
<profile>
<id>admin</id>
<build>
<pluginManagement>
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-keycloak-add-user-json</id>
<phase>process-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${auth.server.home}/standalone/configuration</outputDirectory>
<resources>
<resource>
<directory>${common.resources}</directory>
<includes>
<include>keycloak-add-user.json</include>
</includes>
</resource>
</resources>
<overwrite>true</overwrite>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</profile>
<profile>
<id>auth-server-wildfly</id>
<properties>

1
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java
View file

@ -51,6 +51,7 @@ public final class WaitUtils {
Thread.sleep(millis);
} catch (InterruptedException ex) {
Logger.getLogger(WaitUtils.class.getName()).log(Level.SEVERE, null, ex);
Thread.currentThread().interrupt();
}
}

65
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md
View file

@ -1,2 +1,65 @@
#
# Keycloak Adapter Tests - JBoss Remote
## Performance Tests
### Parameters
* Warmup phase
- `warmup.load` Load during warmup phase (# of clients).
- `warmup.duration` Duration of warmup phase in seconds.
* Measuremet iterations
- `initial.load` Load for the initial measurement iteration (# of clients).
- `load.increase` How many clients to add after each iteration.
- `load.increase.rate` How many clients to add per second.
- `measurement.duration` Duration of measurement iteration (in seconds).
* Limits
- `max.iterations`
- `max.threads`
* Other
- `sleep.between.loops` Sleep period between scenario loops.
### Generated Load
Warmup phase and measurement iterations with load-increase phases in between.
load
^
│
│ /
│ _________/
│ /| |
│ / | |
│ _________/ | |
│ /| | | |
│ / | | | |
│ _________/ | | | |
│ /│ | | | | |
│ / | | | | | |
│ _________/ | | | | | |
│ /| | | | | | | |
│ ____________/ | | | | | | | |
│ /| | | | | | | | | |
│/ | | | | | | | | | |
└──|────────────|─|─────────|──|─────────|──|─────────|──|─────────|───────> time
<--warmup--> <--it.1-> <--it.2-> <--it.3-> <--it.4->
### Login-Logout Test Scenario
#### Collected Statistics
- ACCESS_REQUEST_TIME
- LOGIN_REQUEST_TIME
- LOGIN_VERIFY_REQUEST_TIME
- LOGOUT_REQUEST_TIME
- LOGOUT_VERIFY_REQUEST_TIME
#### Parameters
* Limits
- `max.login.time.average` Maximum accepted average value of LOGIN_REQUEST_TIME.
- `max.logout.time.average` Maximum accepted average value of LOGOUT_REQUEST_TIME.
- `max.timeout.percentage` Maximum accepted timeout percentage for all statistics.

7
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml
View file

@ -49,6 +49,11 @@
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.2</version>
</dependency>
</dependencies>
<build>
@ -157,7 +162,7 @@
<warmup.load>1</warmup.load>
<warmup.duration>10</warmup.duration>
<max.iterations>0</max.iterations>
<sleep.between.repeats>30</sleep.between.repeats>
<sleep.between.loops>30</sleep.between.loops>
</systemPropertyVariables>
</configuration>
</plugin>

18
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutParameters.java
View file

@ -1,18 +0,0 @@
package org.keycloak.testsuite.performance;
/**
*
* @author tkyjovsk
*/
public class LoginLogoutParameters {
public static final Integer AVERAGE_LOGIN_TIME_LIMIT = Integer.parseInt(System.getProperty("average.login.time.limit", "500"));
public static final Integer AVERAGE_LOGOUT_TIME_LIMIT = Integer.parseInt(System.getProperty("average.logout.time.limit", "500"));
public static final String ACCESS_REQUEST_TIME = "ACCESS_REQUEST";
public static final String LOGIN_REQUEST_TIME = "LOGIN_REQUEST";
public static final String LOGIN_VERIFY_REQUEST_TIME = "LOGIN_VERIFY_REQUEST";
public static final String LOGOUT_REQUEST_TIME = "LOGOUT_REQUEST";
public static final String LOGOUT_VERIFY_REQUEST_TIME = "LOGOUT_VERIFY_REQUEST";
}

38
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java Normal file
View file

@ -0,0 +1,38 @@
package org.keycloak.testsuite.performance;
/**
*
* @author tkyjovsk
*/
public class LoginLogoutTestParameters {
// Statistics
public static final String ACCESS_REQUEST_TIME = "ACCESS_REQUEST";
public static final String LOGIN_REQUEST_TIME = "LOGIN_REQUEST";
public static final String LOGIN_VERIFY_REQUEST_TIME = "LOGIN_VERIFY_REQUEST";
public static final String LOGOUT_REQUEST_TIME = "LOGOUT_REQUEST";
public static final String LOGOUT_VERIFY_REQUEST_TIME = "LOGOUT_VERIFY_REQUEST";
// Limits
public static final Integer MAX_LOGIN_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.login.time.average", "500"));
public static final Integer MAX_LOGOUT_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.logout.time.average", "500"));
public static final double MAX_TIMEOUT_PERCENTAGE = Double.parseDouble(System.getProperty("max.timeout.percentage", "0"));
// Other
public static final Integer PASSWORD_HASH_ITERATIONS = Integer.parseInt(System.getProperty("password.hash.iterations", "1"));
public static boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
return isTimeoutPercentageWithinLimits(measurement)
&& measurement.getStatistics().get(LOGIN_REQUEST_TIME).getAverage() < MAX_LOGIN_TIME_AVERAGE
&& measurement.getStatistics().get(LOGOUT_REQUEST_TIME).getAverage() < MAX_LOGOUT_TIME_AVERAGE;
}
public static boolean isTimeoutPercentageWithinLimits(PerformanceMeasurement measurement) {
boolean withinLimits = true;
for (String statistic : measurement.getStatistics().keySet()) {
withinLimits = withinLimits && measurement.getTimeoutPercentage(statistic) <= MAX_TIMEOUT_PERCENTAGE;
}
return withinLimits;
}
}

42
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoopingRunnable.java Normal file
View file

@ -0,0 +1,42 @@
package org.keycloak.testsuite.performance;
import static org.keycloak.testsuite.util.WaitUtils.pause;
/**
*
* @author tkyjovsk
*/
public abstract class LoopingRunnable implements Runnable {
private long sleepBetweenLoopsMillis;
private long loopCounter;
public LoopingRunnable() {
this(0);
this.loopCounter = 0;
}
public LoopingRunnable(long sleepBetweenLoopsMillis) {
this.sleepBetweenLoopsMillis = sleepBetweenLoopsMillis;
}
public void setSleepBetweenLoopsMillis(long sleepBetweenLoopsMillis) {
this.sleepBetweenLoopsMillis = sleepBetweenLoopsMillis;
}
public long getLoopCounter() {
return loopCounter;
}
@Override
public void run() {
while (!Thread.interrupted()) {
loop();
loopCounter++;
pause(sleepBetweenLoopsMillis);
}
}
public abstract void loop();
}

14
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java
View file

@ -6,21 +6,21 @@ package org.keycloak.testsuite.performance;
*/
public class OperationTimeoutException extends Exception {
private final String metric;
private final String statistic;
private final long value;
public OperationTimeoutException(String metric, Throwable cause) {
this(metric, 0, cause);
public OperationTimeoutException(String statistic, Throwable cause) {
this(statistic, 0, cause);
}
public OperationTimeoutException(String metric, long value, Throwable cause) {
public OperationTimeoutException(String statistic, long value, Throwable cause) {
super(cause);
this.metric = metric;
this.statistic = statistic;
this.value = value;
}
public String getMetric() {
return metric;
public String getStatistic() {
return statistic;
}
public long getValue() {

145
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java Normal file
View file

@ -0,0 +1,145 @@
package org.keycloak.testsuite.performance;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import static org.keycloak.testsuite.performance.PerformanceTest.LOG;
import org.keycloak.testsuite.performance.statistics.SimpleStatistics;
import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
/**
*
* @author tkyjovsk
*/
public class PerformanceMeasurement {
private final Date started;
private final int load;
private long durationMillis;
private SimpleStatistics statistics;
private SimpleStatistics timeoutStatistics;
public static final DateFormat ISO8601_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXX"); // should be compatible with `date --iso-8601=seconds`
public static final DateFormat RFC3339_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ssZ"); // should be compatible with `date --rfc-3339=seconds`
public PerformanceMeasurement(int load) {
this.started = new Date();
this.load = load;
}
public SimpleStatistics getStatistics() {
return this.statistics;
}
public SimpleStatistics getTimeoutStatistics() {
return this.timeoutStatistics;
}
public void setStatistics(SimpleStatistics statistics, SimpleStatistics timeoutStatistics) {
this.durationMillis = new Date().getTime() - started.getTime();
if (durationMillis < 0) {
throw new IllegalStateException("Cannot set a negative duration.");
}
this.statistics = statistics;
this.timeoutStatistics = timeoutStatistics;
}
private void checkStatisticsNotNull() {
if (statistics == null || timeoutStatistics == null) {
throw new IllegalStateException("Iteration doesn't have any statistics set.");
}
}
public double getThroughput(String statistic) {
checkStatisticsNotNull();
return (double) statistics.get(statistic).getCount() / durationMillis * 1000;
}
public double getTimeoutPercentage(String statistic) {
checkStatisticsNotNull();
long timeouts = timeoutStatistics.containsKey(statistic) ? timeoutStatistics.get(statistic).getCount() : 0;
return (double) timeouts / statistics.get(statistic).getCount();
}
public static final Object[] HEADER = new String[]{
"Timestamp",
"Load",
"Duration",
"Count",
"Min",
"Max",
"Average",
"Standard Deviation",
"Timeout Percentage",
"Throughput",};
public List toRecord(String statistic) {
checkStatisticsNotNull();
List record = new ArrayList();
record.add(ISO8601_DATE_FORMAT.format(started));
record.add(load);
record.add(durationMillis);
record.add(statistics.get(statistic).getCount());
record.add(statistics.get(statistic).getMin());
record.add(statistics.get(statistic).getMax());
record.add(statistics.get(statistic).getAverage());
record.add(statistics.get(statistic).getStandardDeviation());
record.add(getTimeoutPercentage(statistic));
record.add(getThroughput(statistic));
return record;
}
public void printToCSV() {
printToCSV(null);
}
public void printToCSV(String testName) {
checkStatisticsNotNull();
for (String statistic : statistics.keySet()) {
File csvFile = new File(PROJECT_BUILD_DIRECTORY + "/measurements" + (testName == null ? "" : "/" + testName),
statistic + ".csv");
boolean csvFileCreated = false;
if (!csvFile.exists()) {
try {
csvFile.getParentFile().mkdirs();
csvFileCreated = csvFile.createNewFile();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile, true))) {
CSVPrinter printer = new CSVPrinter(writer, CSVFormat.RFC4180);
if (csvFileCreated) {
printer.printRecord(HEADER);
}
printer.printRecord(toRecord(statistic));
printer.flush();
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}
public void printToLog() {
LOG.info("Measurement results:");
LOG.info("Operation " + Arrays.toString(HEADER));
for (String statistic : statistics.keySet()) {
LOG.info(statistic + " " + toRecord(statistic));
}
}
}

56
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java Normal file
View file

@ -0,0 +1,56 @@
package org.keycloak.testsuite.performance;
import java.util.concurrent.ConcurrentHashMap;
import org.keycloak.testsuite.performance.statistics.DataHoldingUpdatableStatistic;
import org.keycloak.testsuite.performance.statistics.MovingUpdatableStatistic;
import org.keycloak.testsuite.performance.statistics.SimpleStatistics;
import org.keycloak.testsuite.performance.statistics.UpdatableStatistics;
import org.keycloak.testsuite.performance.statistics.UpdatableStatistic;
/**
* PerformanceStatistics. Concurrent hash map of UpdatableStatistic objects,
* type of which can be selected by the "statistic.type" property.
*
* @author tkyjovsk
*/
public class PerformanceStatistics extends ConcurrentHashMap<String, UpdatableStatistic> implements UpdatableStatistics {
public static final String STATISTIC_TYPE = System.getProperty("statistic.type", MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE);
@Override
public void reset() {
clear();
}
private UpdatableStatistic createIfNullAndGet(String statistic) {
UpdatableStatistic updatableStatistic = get(statistic);
if (updatableStatistic == null) {
switch (STATISTIC_TYPE) {
case DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE:
updatableStatistic = new DataHoldingUpdatableStatistic();
break;
case MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE:
updatableStatistic = new DataHoldingUpdatableStatistic();
break;
default:
throw new IllegalStateException(String.format(
"Unknown statistic type: '%s'. Supported values: %s | %s",
STATISTIC_TYPE,
DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE,
MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE));
}
put(statistic, updatableStatistic);
}
return updatableStatistic;
}
@Override
public void addValue(String statistic, long value) {
createIfNullAndGet(statistic).addValue(value);
}
public SimpleStatistics snapshot() {
return new SimpleStatistics(this);
}
}

114
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java
View file

@ -11,8 +11,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.keycloak.testsuite.adapter.AbstractExampleAdapterTest;
import org.keycloak.testsuite.performance.metrics.impl.Results;
import org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput;
import static org.keycloak.testsuite.util.WaitUtils.pause;
/**
@ -22,7 +20,7 @@ import static org.keycloak.testsuite.util.WaitUtils.pause;
*/
public abstract class PerformanceTest extends AbstractExampleAdapterTest {
private final Logger LOG = Logger.getLogger(PerformanceTest.class);
public static final Logger LOG = Logger.getLogger(PerformanceTest.class);
public static final Integer WARMUP_LOAD = Integer.parseInt(System.getProperty("warmup.load", "5"));
public static final Integer WARMUP_DURATION = Integer.parseInt(System.getProperty("warmup.duration", "30"));
@ -36,19 +34,20 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
public static final Integer MAX_ITERATIONS = Integer.parseInt(System.getProperty("max.iterations", "10"));
public static final Integer MAX_THREADS = Integer.parseInt(System.getProperty("max.threads", "1000"));
public static final Integer SLEEP_BETWEEN_REPEATS = Integer.parseInt(System.getProperty("sleep.between.repeats", "0"));
public static final Integer SLEEP_BETWEEN_LOOPS = Integer.parseInt(System.getProperty("sleep.between.loops", "0"));
public static final Integer THREADPOOL_TERMINATION_TIMEOUT = Integer.parseInt(System.getProperty("threadpool.termination.timeout", "10"));
public static final Integer ADDITIONAL_SLEEP_AFTER = Integer.parseInt(System.getProperty("additional.sleep.after", "0"));
private final double AVERAGE_TIMEOUT_PERCENTAGE_LIMIT = Double.parseDouble(System.getProperty("average.timeout.percentage.limit", "0.01"));
public static final String SCENARIO_TIME = "SCENARIO";
private int currentLoad;
private ExecutorService executorService;
protected PerformanceTestMetrics metrics = new PerformanceTestMetrics();
protected PerformanceTestMetrics timeouts = new PerformanceTestMetrics();
protected PerformanceStatistics statistics = new PerformanceStatistics();
protected PerformanceStatistics timeoutStatistics = new PerformanceStatistics(); // for keeping track of # of conn. timeout exceptions
protected List<ResultsWithThroughput> resultsList = new ArrayList<>();
protected List<ResultsWithThroughput> timeoutResultsList = new ArrayList<>();
protected List<PerformanceMeasurement> measurements = new ArrayList<>();
@Before
public void before() {
@ -59,14 +58,20 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
executorService = Executors.newFixedThreadPool(MAX_THREADS);
currentLoad = 0;
metrics.clear();
statistics.clear();
timeoutStatistics.clear();
}
@After
public void after() throws IOException, InterruptedException {
executorService.shutdown();
LOG.info("Waiting for threadpool termination.");
executorService.awaitTermination(10, TimeUnit.SECONDS);
executorService.awaitTermination(THREADPOOL_TERMINATION_TIMEOUT, TimeUnit.SECONDS);
pause(ADDITIONAL_SLEEP_AFTER * 1000);
LOG.info("Logging out all sessions.");
testRealmResource().logoutAll();
}
@Test
@ -79,7 +84,7 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
int loadIncrease = (i == 0)
? INITIAL_LOAD - WARMUP_LOAD // increase from warmup to initial load
: LOAD_INCREASE; // increase load between iterations
: LOAD_INCREASE; // increase load between measurements
increaseLoadBy(loadIncrease);
measurePerformance();
@ -88,8 +93,8 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
LOG.warn("Threadpool capacity reached. Stopping the test.");
break;
}
if (!isLatestResultsWithinLimits()) {
LOG.warn("The latest measurement surpassed expected limit. Stopping the test.");
if (!isLatestMeasurementWithinLimits()) {
LOG.warn("The latest measurement exceeded expected limit. Stopping the test.");
break;
}
}
@ -113,7 +118,7 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
throw new IllegalArgumentException("Cannot increase load beyond threadpool capacity (" + MAX_THREADS + ").");
}
if (loadIncrease > 0) {
LOG.info(String.format("Increasing load from %s to %s.", currentLoad, currentLoad + loadIncrease));
LOG.info(String.format("Increasing load from %s to %s at +%s clients/s.", currentLoad, currentLoad + loadIncrease, LOAD_INCREASE_RATE));
for (int t = 0; t < loadIncrease; t++) {
executorService.submit(newRunnable());
currentLoad++;
@ -123,26 +128,21 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
}
private void measurePerformance() {
LOG.info("Measuring performance");
LOG.info("Iteration: " + (resultsList.size() + 1));
LOG.info("Duration: " + MEASUREMENT_DURATION + " s");
LOG.info("Load: " + currentLoad);
PerformanceMeasurement measurement = new PerformanceMeasurement(currentLoad);
statistics.reset();
timeoutStatistics.reset();
LOG.info(String.format("Measuring performance. Iteration: %s, Load: %s, Duration: %s s", measurements.size() + 1, currentLoad, MEASUREMENT_DURATION));
metrics.reset();
pauseWithErrorChecking(MEASUREMENT_DURATION * 1000);
resultsList.add(metrics.computeMetrics());
timeoutResultsList.add(timeouts.computeMetrics());
getLatestResults().logResults(); // to file
LOG.info("Timeouts: " + getLatestTimeoutResults());
}
measurement.setStatistics(
statistics.snapshot(),
timeoutStatistics.snapshot());
measurements.add(measurement);
protected ResultsWithThroughput getLatestResults() {
return resultsList.isEmpty() ? null : resultsList.get(resultsList.size() - 1);
}
protected Results getLatestTimeoutResults() {
return timeoutResultsList.isEmpty() ? null : timeoutResultsList.get(timeoutResultsList.size() - 1);
measurement.printToCSV(getTestName());
measurement.printToLog();
}
private Throwable error = null;
@ -159,12 +159,14 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
pauseWithErrorChecking(millis, 1000);
}
protected void pauseWithErrorChecking(long millis, long checkIntervals) {
long count = millis / checkIntervals;
long remainder = millis % checkIntervals;
for (int i = 0; i < count + 1; i++) { // repeat 'count' times + once for remainder
if (i < count || remainder > 0) { // on last iteration check if any remainder
pause(checkIntervals);
protected void pauseWithErrorChecking(long millis, long checkDurationMillis) {
long checkDurationMillisMin = Math.min(millis, checkDurationMillis);
long checkCount = millis / checkDurationMillis;
long remainder = millis % checkDurationMillis;
LOG.debug(String.format("Pause %s ms, checking errors once per %s ms", millis, checkDurationMillisMin));
for (int i = 0; i < checkCount + 1; i++) { // loop 'count' times + once for remainder
if (i < checkCount || remainder > 0) { // on last iteration check if any remainder
pause(checkDurationMillisMin);
if (getError() != null) {
throw new RuntimeException("PerformanceTestRunnable threw an exception. Stopping the test.", getError());
}
@ -172,36 +174,38 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
}
}
protected abstract boolean isLatestResultsWithinLimits();
protected boolean isLatestTimeoutsWithinLimits() {
boolean socketTimeoutsWithinLimits = true;
for (String metric : getLatestResults().keySet()) {
long timemoutCount = getLatestTimeoutResults().containsKey(metric) ? getLatestTimeoutResults().get(metric).getCount() : 0;
double timeoutPercentage = (double) timemoutCount / getLatestResults().get(metric).getCount();
socketTimeoutsWithinLimits = socketTimeoutsWithinLimits && timeoutPercentage < AVERAGE_TIMEOUT_PERCENTAGE_LIMIT;
}
return socketTimeoutsWithinLimits;
protected PerformanceMeasurement getLatestMeasurement() {
return measurements.isEmpty() ? null : measurements.get(measurements.size() - 1);
}
protected boolean isLatestMeasurementWithinLimits() {
return isMeasurementWithinLimits(getLatestMeasurement());
}
protected abstract boolean isMeasurementWithinLimits(PerformanceMeasurement measurement);
protected abstract Runnable newRunnable();
public abstract class Runnable extends RepeatRunnable {
public abstract class Runnable extends LoopingRunnable {
protected final Timer timer;
protected final Timer timer; // for timing individual operations/requests
private final Timer scenarioTimer; // for timing the whole scenario
public Runnable() {
super(SLEEP_BETWEEN_REPEATS * 1000);
super(SLEEP_BETWEEN_LOOPS * 1000);
this.timer = new Timer();
this.scenarioTimer = new Timer();
}
@Override
public void repeat() {
public void loop() {
try {
scenarioTimer.reset();
performanceScenario();
statistics.addValue(SCENARIO_TIME, scenarioTimer.getElapsedTime());
} catch (OperationTimeoutException ex) {
timeouts.addValue(ex.getMetric(), ex.getValue());
LOG.debug(String.format("Operatin %s timed out. Cause: %s.", ex.getMetric(), ex.getCause()));
timeoutStatistics.addValue(ex.getStatistic(), ex.getValue());
LOG.debug(String.format("Operation %s timed out. Cause: %s.", ex.getStatistic(), ex.getCause()));
} catch (AssertionError | Exception ex) {
setError(ex);
throw new RuntimeException(ex);
@ -212,4 +216,8 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest {
}
public String getTestName() {
return this.getClass().getSimpleName();
}
}

51
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTestMetrics.java
View file

@ -1,51 +0,0 @@
package org.keycloak.testsuite.performance;
import java.util.concurrent.ConcurrentHashMap;
import org.keycloak.testsuite.performance.metrics.ComputedMetric;
import org.keycloak.testsuite.performance.metrics.ComputedMetrics;
import org.keycloak.testsuite.performance.metrics.impl.ArrayListMetric;
import org.keycloak.testsuite.performance.metrics.impl.MovingAverageMetric;
import org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput;
/**
*
* @author tkyjovsk
*/
public class PerformanceTestMetrics extends ConcurrentHashMap<String,ComputedMetric> implements ComputedMetrics {
public static final String METRIC_MOVING_AVERAGE = "MovingAverage";
public static final String METRIC_ARRAY_LIST = "ArrayList";
public static final String METRIC = System.getProperty("metric", METRIC_MOVING_AVERAGE);
Timer timer = new Timer();
@Override
public void reset() {
clear();
timer.reset();
}
private ComputedMetric getOrCreate(String metric) {
ComputedMetric m = get(metric);
if (m == null) {
if (METRIC_ARRAY_LIST.equals(metric)) {
m = new ArrayListMetric();
} else {
m = new MovingAverageMetric();
}
put(metric, m);
}
return m;
}
@Override
public void addValue(String metric, long value) {
getOrCreate(metric).addValue(value);
}
@Override
public ResultsWithThroughput computeMetrics() {
return new ResultsWithThroughput(this, timer.getElapsedTime());
}
}

42
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/RepeatRunnable.java
View file

@ -1,42 +0,0 @@
package org.keycloak.testsuite.performance;
import static org.keycloak.testsuite.util.WaitUtils.pause;
/**
*
* @author tkyjovsk
*/
public abstract class RepeatRunnable implements Runnable {
private long sleepBetweenRepeatsMillis;
private long repeatCounter;
public RepeatRunnable() {
this(0);
this.repeatCounter = 0;
}
public RepeatRunnable(long sleepBetweenRepeatsMillis) {
this.sleepBetweenRepeatsMillis = sleepBetweenRepeatsMillis;
}
public void setSleepBetweenRepeatsMillis(long sleepBetweenRepeatsMillis) {
this.sleepBetweenRepeatsMillis = sleepBetweenRepeatsMillis;
}
public long getRepeatCounter() {
return repeatCounter;
}
@Override
public void run() {
while (true) {
repeat();
repeatCounter++;
pause(sleepBetweenRepeatsMillis);
}
}
public abstract void repeat();
}

40
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java
View file

@ -13,18 +13,19 @@ import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
import org.keycloak.testsuite.performance.page.AppProfileJEE;
import org.openqa.selenium.By;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME;
import org.keycloak.testsuite.performance.PerformanceTest;
import static org.junit.Assert.assertTrue;
import org.keycloak.testsuite.performance.OperationTimeoutException;
import static org.keycloak.testsuite.util.IOUtil.loadRealm;
import org.openqa.selenium.TimeoutException;
import org.keycloak.testsuite.performance.PerformanceMeasurement;
import org.keycloak.testsuite.performance.LoginLogoutTestParameters;
import static org.junit.Assert.assertTrue;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS;
import static org.keycloak.testsuite.util.IOUtil.loadRealm;
/**
*
@ -65,7 +66,9 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
@Override
public void addAdapterTestRealms(List<RealmRepresentation> testRealms) {
testRealms.add(loadRealm("/examples-realm.json"));
RealmRepresentation examplesRealm = loadRealm("/examples-realm.json");
examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")");
testRealms.add(examplesRealm);
}
@Before
@ -83,16 +86,15 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
}
@Override
protected boolean isLatestResultsWithinLimits() {
return getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT
&& getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT;
protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement);
}
public class Runnable extends HtmlUnitPerformanceTest.Runnable {
@Override
public void performanceScenario() throws Exception {
LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter()));
LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter()));
driver.manage().deleteAllCookies();
// ACCESS
@ -104,7 +106,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex);
}
metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
// LOGIN
LOG.trace("Logging in");
@ -119,7 +121,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex);
}
metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGIN
LOG.trace("Verifying login");
@ -130,7 +132,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex);
}
metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
// LOGOUT
LOG.trace("Logging out");
@ -141,7 +143,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex);
}
metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGOUT
LOG.trace("Verifying logout");
@ -152,7 +154,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest {
} catch (TimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex);
}
metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
LOG.trace("Logged out");
}

63
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java
View file

@ -27,18 +27,19 @@ import org.junit.Before;
import org.keycloak.representations.idm.RealmRepresentation;
import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
import org.keycloak.testsuite.performance.page.AppProfileJEE;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME;
import org.keycloak.testsuite.performance.PerformanceTest;
import org.keycloak.testsuite.performance.OperationTimeoutException;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME;
import org.keycloak.testsuite.performance.PerformanceMeasurement;
import org.keycloak.testsuite.performance.LoginLogoutTestParameters;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS;
import static org.keycloak.testsuite.util.IOUtil.loadRealm;
/**
@ -80,7 +81,9 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
@Override
public void addAdapterTestRealms(List<RealmRepresentation> testRealms) {
testRealms.add(loadRealm("/examples-realm.json"));
RealmRepresentation examplesRealm = loadRealm("/examples-realm.json");
examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")");
testRealms.add(examplesRealm);
}
@Before
@ -98,17 +101,15 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
}
@Override
protected boolean isLatestResultsWithinLimits() {
return isLatestTimeoutsWithinLimits()
&& getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT
&& getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT;
protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) {
return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement);
}
public class Runnable extends HttpClientPerformanceTest.Runnable {
@Override
public void performanceScenario() throws IOException, OperationTimeoutException {
LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter()));
LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter()));
context.getCookieStore().clear();
// ACCESS
@ -118,17 +119,17 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
assertEquals("ACCESS_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
assertEquals(1, context.getRedirectLocations().size());
assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl));
assertEquals("ACCESS_REQUEST has 1 redirect", 1, context.getRedirectLocations().size());
assertTrue("ACCESS_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl));
pageContent = EntityUtils.toString(r.getEntity());
} catch (SocketException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex.bytesTransferred, ex);
}
metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime());
// LOGIN
final HttpPost loginRequest = new HttpPost(getLoginUrlFromPage(pageContent));
@ -141,31 +142,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(loginRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(loginRequest, context)) {
assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
assertEquals("LOGIN_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
assertEquals(2, context.getRedirectLocations().size());
assertTrue(getLastRedirect().toASCIIString().equals(securedUrl));
assertEquals("LOGIN_REQUEST has 2 redirects", 2, context.getRedirectLocations().size());
assertTrue("LOGIN_REQUEST redirects to secured page", getLastRedirect().toASCIIString().equals(securedUrl));
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex.bytesTransferred, ex);
}
metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGIN
LOG.trace("Verifying login");
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
assertEquals("LOGIN_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
assertEquals(0, context.getRedirectLocations().size());
assertEquals("LOGIN_VERIFY_REQUEST has 0 redirects", 0, context.getRedirectLocations().size());
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex);
}
metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime());
// LOGOUT
final HttpGet logoutRequest = new HttpGet(logoutUrl);
@ -173,31 +174,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest {
LOG.trace(logoutRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(logoutRequest, context)) {
assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
assertEquals("LOGOUT_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
assertEquals(0, context.getRedirectLocations().size());
assertEquals("LOGOUT_REQUEST has 0 redirects", 0, context.getRedirectLocations().size());
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex.bytesTransferred, ex);
}
metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime());
// VERIFY LOGOUT
LOG.trace("Verifying logout");
LOG.trace(getSecuredPageRequest);
timer.reset();
try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) {
assertEquals(HTTP_OK, r.getStatusLine().getStatusCode());
assertEquals("LOGOUT_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode());
logRedirects();
assertEquals(1, context.getRedirectLocations().size());
assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl));
assertEquals("LOGOUT_VERIFY_REQUEST has 1 redirect", 1, context.getRedirectLocations().size());
assertTrue("LOGOUT_VERIFY_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl));
} catch (SocketException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex);
} catch (SocketTimeoutException ex) {
throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex);
}
metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime());
LOG.trace("Logged out");

2
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java
View file

@ -12,6 +12,7 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@ -45,6 +46,7 @@ public abstract class HttpClientPerformanceTest extends PerformanceTest {
.setDefaultCookieStore(new BasicCookieStore())
.setDefaultRequestConfig(getDefaultRequestConfig())
.setRedirectStrategy(new CustomRedirectStrategy())
.setRetryHandler(new DefaultHttpRequestRetryHandler(0, false))
.build();
}

12
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetric.java
View file

@ -1,12 +0,0 @@
package org.keycloak.testsuite.performance.metrics;
/**
*
* @author tkyjovsk
*/
public interface ComputedMetric extends Metric {
public void reset();
public void addValue(long value);
}

13
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetrics.java
View file

@ -1,13 +0,0 @@
package org.keycloak.testsuite.performance.metrics;
/**
*
* @author tkyjovsk
*/
public interface ComputedMetrics extends Metrics<ComputedMetric> {
public void reset();
public void addValue(String metric, long value);
public Metrics computeMetrics();
}

15
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metric.java
View file

@ -1,15 +0,0 @@
package org.keycloak.testsuite.performance.metrics;
/**
*
* @author tkyjovsk
*/
public interface Metric {
public long getCount();
public long getMin();
public long getMax();
public double getAverage();
public double getStandardDeviation();
}

12
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metrics.java
View file

@ -1,12 +0,0 @@
package org.keycloak.testsuite.performance.metrics;
import java.util.Map;
/**
*
* @author tkyjovsk
* @param <T>
*/
public interface Metrics<T extends Metric> extends Map<String, T> {
}

58
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ArrayListMetric.java
View file

@ -1,58 +0,0 @@
package org.keycloak.testsuite.performance.metrics.impl;
import java.util.ArrayList;
import java.util.Collections;
import org.keycloak.testsuite.performance.metrics.ComputedMetric;
/**
*
* @author tkyjovsk
*/
public class ArrayListMetric extends ArrayList<Long> implements ComputedMetric {
@Override
public void reset() {
clear();
}
@Override
public void addValue(long value) {
add(value);
}
@Override
public long getCount() {
return size();
}
@Override
public long getMin() {
return Collections.min(this);
}
@Override
public long getMax() {
return Collections.max(this);
}
@Override
public double getAverage() {
long sum = 0;
for (long l : this) {
sum += l;
}
return isEmpty() ? 0 : sum / size();
}
@Override
public double getStandardDeviation() {
double average = getAverage();
long sumSquare = 0;
for (long l : this) {
sumSquare += l * l;
}
return isEmpty() ? 0
: Math.sqrt(sumSquare / size() - (average * average));
}
}

23
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Results.java
View file

@ -1,23 +0,0 @@
package org.keycloak.testsuite.performance.metrics.impl;
import java.util.TreeMap;
import org.keycloak.testsuite.performance.metrics.ComputedMetrics;
import org.keycloak.testsuite.performance.metrics.Metrics;
/**
*
* @author tkyjovsk
*/
public class Results extends TreeMap<String, Result> implements Metrics<Result> {
public Results(ComputedMetrics metrics) {
for (String metric : metrics.keySet()) {
put(metric, new Result(metrics.get(metric)));
}
}
public String getHeader() {
return "# Operation Count Min Max Average Standard-Deviation";
}
}

48
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ResultsWithThroughput.java
View file

@ -1,48 +0,0 @@
package org.keycloak.testsuite.performance.metrics.impl;
import org.jboss.logging.Logger;
import java.util.HashMap;
import java.util.Map;
import org.keycloak.testsuite.performance.metrics.ComputedMetrics;
/**
*
* @author tkyjovsk
*/
public class ResultsWithThroughput extends Results {
public static final Logger LOG = Logger.getLogger(ResultsWithThroughput.class);
private final Map<String, Double> throughput;
public ResultsWithThroughput(ComputedMetrics metrics, long durationMillis) {
super(metrics);
throughput = new HashMap<>();
for (String metric : keySet()) {
throughput.put(metric, (double) get(metric).getCount() / durationMillis * 1000);
}
}
public Map<String, Double> getThroughput() {
return throughput;
}
@Override
public String toString() {
return "Results: " + super.toString() + "\n"
+ "Throughput: " + getThroughput();
}
@Override
public String getHeader() {
return super.getHeader() + " Throughput";
}
public void logResults() {
LOG.info(getHeader());
for (String metric : keySet()) {
LOG.info(metric + " " + get(metric).toLogString() + " " + getThroughput().get(metric));
}
}
}

62
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/DataHoldingUpdatableStatistic.java Normal file
View file

@ -0,0 +1,62 @@
package org.keycloak.testsuite.performance.statistics;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
*
* @author tkyjovsk
*/
public class DataHoldingUpdatableStatistic implements UpdatableStatistic {
public static final String STATISTIC_TYPE_PROPERTY_VALUE = "data";
private final List<Long> data = Collections.synchronizedList(new ArrayList<Long>());
@Override
public synchronized void reset() {
data.clear();
}
@Override
public synchronized void addValue(long value) {
data.add(value);
}
@Override
public synchronized long getCount() {
return data.size();
}
@Override
public synchronized long getMin() {
return Collections.min(data);
}
@Override
public synchronized long getMax() {
return Collections.max(data);
}
@Override
public synchronized double getAverage() {
long sum = 0;
for (long l : data) {
sum += l;
}
return data.isEmpty() ? 0 : sum / data.size();
}
@Override
public synchronized double getStandardDeviation() {
double average = getAverage();
long sumSquare = 0;
for (long l : data) {
sumSquare += l * l;
}
return data.isEmpty() ? 0
: Math.sqrt(sumSquare / data.size() - (average * average));
}
}

34
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/MovingAverageMetric.java → testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/MovingUpdatableStatistic.java
View file

@ -1,13 +1,15 @@
package org.keycloak.testsuite.performance.metrics.impl;
package org.keycloak.testsuite.performance.statistics;
import java.math.BigDecimal;
import org.keycloak.testsuite.performance.metrics.ComputedMetric;
/**
* Allows to compute statistical values without holding the actual measurements.
*
* @author tkyjovsk
*/
public final class MovingAverageMetric implements ComputedMetric {
public final class MovingUpdatableStatistic implements UpdatableStatistic {
public static final String STATISTIC_TYPE_PROPERTY_VALUE = "moving";
private BigDecimal sum;
private BigDecimal sumSquare;
@ -15,7 +17,7 @@ public final class MovingAverageMetric implements ComputedMetric {
private long min;
private long max;
public MovingAverageMetric() {
public MovingUpdatableStatistic() {
reset();
}
@ -29,17 +31,26 @@ public final class MovingAverageMetric implements ComputedMetric {
}
@Override
public long getCount() {
public synchronized void addValue(long value) {
sum = sum.add(new BigDecimal(value));
sumSquare = sumSquare.add(new BigDecimal(value * value));
min = Math.min(min, value);
max = Math.max(max, value);
count++;
}
@Override
public synchronized long getCount() {
return count;
}
@Override
public long getMin() {
public synchronized long getMin() {
return min;
}
@Override
public long getMax() {
public synchronized long getMax() {
return max;
}
@ -56,15 +67,6 @@ public final class MovingAverageMetric implements ComputedMetric {
: Math.sqrt(sumSquare.longValue() / count - (average * average));
}
@Override
public synchronized void addValue(long value) {
sum = sum.add(new BigDecimal(value));
sumSquare = sumSquare.add(new BigDecimal(value * value));
min = Math.min(min, value);
max = Math.max(max, value);
count++;
}
@Override
public String toString() {
return Double.toString(getAverage());

20
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Result.java → testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistic.java
View file

@ -1,14 +1,14 @@
package org.keycloak.testsuite.performance.metrics.impl;
package org.keycloak.testsuite.performance.statistics;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.keycloak.testsuite.performance.metrics.Metric;
/**
* SimpleStatistic just holds data.
*
* @author tkyjovsk
*/
public class Result implements Metric {
public class SimpleStatistic implements Statistic {
private final long count;
private final long min;
@ -16,7 +16,7 @@ public class Result implements Metric {
private final double average;
private final double standardDeviation;
public Result(long count, long min, long max, double average, double standardDeviation) {
public SimpleStatistic(long count, long min, long max, double average, double standardDeviation) {
this.count = count;
this.min = min;
this.max = max;
@ -24,13 +24,13 @@ public class Result implements Metric {
this.standardDeviation = standardDeviation;
}
public Result(Metric metric) {
public SimpleStatistic(Statistic statistic) {
this(
metric.getCount(),
metric.getMin(),
metric.getMax(),
metric.getAverage(),
metric.getStandardDeviation());
statistic.getCount(),
statistic.getMin(),
statistic.getMax(),
statistic.getAverage(),
statistic.getStandardDeviation());
}
@Override

17
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java Normal file
View file

@ -0,0 +1,17 @@
package org.keycloak.testsuite.performance.statistics;
import java.util.TreeMap;
/**
*
* @author tkyjovsk
*/
public class SimpleStatistics extends TreeMap<String, SimpleStatistic> implements Statistics<SimpleStatistic> {
public SimpleStatistics(Statistics statistics) {
for (Object statistic : statistics.keySet()) {
put(statistic.toString(), new SimpleStatistic((Statistic) statistics.get(statistic)));
}
}
}

17
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistic.java Normal file
View file

@ -0,0 +1,17 @@
package org.keycloak.testsuite.performance.statistics;
/**
* Statistic provides statistical information about a data set.
* Number of measurements, minimum/maximum/average value and standard deviation.
*
* @author tkyjovsk
*/
public interface Statistic {
public long getCount();
public long getMin();
public long getMax();
public double getAverage();
public double getStandardDeviation();
}

13
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistics.java Normal file
View file

@ -0,0 +1,13 @@
package org.keycloak.testsuite.performance.statistics;
import java.util.Map;
/**
* A Map of named statistics.
*
* @author tkyjovsk
* @param <T>
*/
public interface Statistics<T extends Statistic> extends Map<String, T> {
}

15
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistic.java Normal file
View file

@ -0,0 +1,15 @@
package org.keycloak.testsuite.performance.statistics;
/**
* UpdatableStatistic. A Statistic that can be updated, e.g. from PerformanceTest.Runnable.
* Implementations should be thread-safe.
*
* @author tkyjovsk
*/
public interface UpdatableStatistic extends Statistic {
public void reset();
public void addValue(long value);
}

13
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistics.java Normal file
View file

@ -0,0 +1,13 @@
package org.keycloak.testsuite.performance.statistics;
/**
*
* @author tkyjovsk
*/
public interface UpdatableStatistics extends Statistics<UpdatableStatistic> {
public void reset();
public void addValue(String statistic, long value);
}

12
testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties
View file

@ -8,6 +8,7 @@ log4j.appender.DEFAULT.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p [%c] [%t]
log4j.logger.org.keycloak=OFF
log4j.logger.org.keycloak.testsuite=INFO
log4j.logger.org.keycloak.testsuite.performance.PerformanceTest=INFO
# HtmlUnit
log4j.logger.org.keycloak.testsuite.performance.htmlunit.HtmlUnitLoginLogoutPerfTest=${logging.loginlogout}
@ -17,14 +18,3 @@ log4j.logger.com.gargoylesoftware.htmlunit=OFF
log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientLoginLogoutPerfTest=${logging.loginlogout}
log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientPerformanceTest$CustomRedirectStrategy=OFF
# RESULTS
log4j.appender.RESULTS=org.apache.log4j.FileAppender
log4j.appender.RESULTS.file=target/results.log
log4j.appender.RESULTS.immediateFlush=true
log4j.appender.RESULTS.append=true
log4j.appender.RESULTS.layout=org.apache.log4j.PatternLayout
log4j.appender.RESULTS.layout.ConversionPattern=%d{HHmmss} %m%n
log4j.logger.org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput=INFO, RESULTS