diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json b/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json new file mode 100644 index 0000000000..60c0f098d1 --- /dev/null +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/keycloak-add-user.json @@ -0,0 +1,15 @@ +[ { + "realm" : "master", + "users" : [ { + "username" : "admin", + "enabled" : true, + "credentials" : [ { + "type" : "password", + "hashedSaltedValue" : "dqalJHLkWhUJZO/q6+z1fvXOohTcGCXcvoU8xCEyvTxGN4wmLx7DtyhKuefggh6Bkx1I2eBTEX4tiWggwyXMDw==", + "salt" : "3fBAt5GAGGxFrV9fznpZHQ==", + "hashIterations" : 100000, + "algorithm" : "pbkdf2" + } ], + "realmRoles" : [ "admin" ] + } ] +} ] \ No newline at end of file diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl b/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl index c5d983d50d..685211b2cb 100644 --- a/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/common/mod_cluster.xsl @@ -22,8 +22,26 @@ - - + + \ + + + + + + + + + + + + + + + + + + diff --git a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml index 02f768db93..ef8a64d230 100644 --- a/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml +++ b/testsuite/integration-arquillian/servers/auth-server/jboss/pom.xml @@ -406,60 +406,60 @@ - - org.apache.maven.plugins - maven-antrun-plugin - 1.8 - - - inject-truststore-into-keycloak-server-json - process-resources - - run - - - - - - - - - - - - - ant-contrib - ant-contrib - 1.0b3 - - - ant - ant - - - - - org.apache.ant - ant-apache-bsf - 1.9.3 - - - org.apache.bsf - bsf-api - 3.1 - - - rhino - js - 1.7R2 - - - org.keycloak - keycloak-core - ${project.version} - - - + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + inject-truststore-into-keycloak-server-json + process-resources + + run + + + + + + + + + + + + + ant-contrib + ant-contrib + 1.0b3 + + + ant + ant + + + + + org.apache.ant + ant-apache-bsf + 1.9.3 + + + org.apache.bsf + bsf-api + 3.1 + + + rhino + js + 1.7R2 + + + org.keycloak + keycloak-core + ${project.version} + + + @@ -615,6 +615,12 @@ 1 1 1 + + simple + + @@ -701,6 +707,12 @@ ${common.resources}/mod_cluster.xsl ${auth.server.home}/standalone/configuration + + + load.metric + ${load.metric} + + @@ -711,6 +723,40 @@ + + + admin + + + + + maven-resources-plugin + + + copy-keycloak-add-user-json + process-resources + + copy-resources + + + ${auth.server.home}/standalone/configuration + + + ${common.resources} + + keycloak-add-user.json + + + + true + + + + + + + + auth-server-wildfly diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java index c81a22c007..5124377871 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/WaitUtils.java @@ -51,6 +51,7 @@ public final class WaitUtils { Thread.sleep(millis); } catch (InterruptedException ex) { Logger.getLogger(WaitUtils.class.getName()).log(Level.SEVERE, null, ex); + Thread.currentThread().interrupt(); } } diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md index a9bd7fde83..40926cb1e9 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/README.md @@ -1,2 +1,65 @@ -# +# Keycloak Adapter Tests - JBoss Remote + +## Performance Tests + +### Parameters + +* Warmup phase + - `warmup.load` Load during warmup phase (# of clients). + - `warmup.duration` Duration of warmup phase in seconds. +* Measuremet iterations + - `initial.load` Load for the initial measurement iteration (# of clients). + - `load.increase` How many clients to add after each iteration. + - `load.increase.rate` How many clients to add per second. + - `measurement.duration` Duration of measurement iteration (in seconds). +* Limits + - `max.iterations` + - `max.threads` +* Other + - `sleep.between.loops` Sleep period between scenario loops. + +### Generated Load + +Warmup phase and measurement iterations with load-increase phases in between. + + load + + ^ + │ + │ / + │ _________/ + │ /| | + │ / | | + │ _________/ | | + │ /| | | | + │ / | | | | + │ _________/ | | | | + │ /│ | | | | | + │ / | | | | | | + │ _________/ | | | | | | + │ /| | | | | | | | + │ ____________/ | | | | | | | | + │ /| | | | | | | | | | + │/ | | | | | | | | | | + └──|────────────|─|─────────|──|─────────|──|─────────|──|─────────|───────> time + + <--warmup--> <--it.1-> <--it.2-> <--it.3-> <--it.4-> + + +### Login-Logout Test Scenario + +#### Collected Statistics + + - ACCESS_REQUEST_TIME + - LOGIN_REQUEST_TIME + - LOGIN_VERIFY_REQUEST_TIME + - LOGOUT_REQUEST_TIME + - LOGOUT_VERIFY_REQUEST_TIME + +#### Parameters + +* Limits + - `max.login.time.average` Maximum accepted average value of LOGIN_REQUEST_TIME. + - `max.logout.time.average` Maximum accepted average value of LOGOUT_REQUEST_TIME. + - `max.timeout.percentage` Maximum accepted timeout percentage for all statistics. diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml index 03ee852bb5..7d2419c4b5 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/pom.xml @@ -49,6 +49,11 @@ commons-io 2.4 + + org.apache.commons + commons-csv + 1.2 + @@ -157,7 +162,7 @@ 1 10 0 - 30 + 30 @@ -166,4 +171,4 @@ - \ No newline at end of file + diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutParameters.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutParameters.java deleted file mode 100644 index 0ca09d2994..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutParameters.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.keycloak.testsuite.performance; - -/** - * - * @author tkyjovsk - */ -public class LoginLogoutParameters { - - public static final Integer AVERAGE_LOGIN_TIME_LIMIT = Integer.parseInt(System.getProperty("average.login.time.limit", "500")); - public static final Integer AVERAGE_LOGOUT_TIME_LIMIT = Integer.parseInt(System.getProperty("average.logout.time.limit", "500")); - - public static final String ACCESS_REQUEST_TIME = "ACCESS_REQUEST"; - public static final String LOGIN_REQUEST_TIME = "LOGIN_REQUEST"; - public static final String LOGIN_VERIFY_REQUEST_TIME = "LOGIN_VERIFY_REQUEST"; - public static final String LOGOUT_REQUEST_TIME = "LOGOUT_REQUEST"; - public static final String LOGOUT_VERIFY_REQUEST_TIME = "LOGOUT_VERIFY_REQUEST"; - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java new file mode 100644 index 0000000000..70dbaec899 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoginLogoutTestParameters.java @@ -0,0 +1,38 @@ +package org.keycloak.testsuite.performance; + +/** + * + * @author tkyjovsk + */ +public class LoginLogoutTestParameters { + + // Statistics + public static final String ACCESS_REQUEST_TIME = "ACCESS_REQUEST"; + public static final String LOGIN_REQUEST_TIME = "LOGIN_REQUEST"; + public static final String LOGIN_VERIFY_REQUEST_TIME = "LOGIN_VERIFY_REQUEST"; + public static final String LOGOUT_REQUEST_TIME = "LOGOUT_REQUEST"; + public static final String LOGOUT_VERIFY_REQUEST_TIME = "LOGOUT_VERIFY_REQUEST"; + + // Limits + public static final Integer MAX_LOGIN_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.login.time.average", "500")); + public static final Integer MAX_LOGOUT_TIME_AVERAGE = Integer.parseInt(System.getProperty("max.logout.time.average", "500")); + public static final double MAX_TIMEOUT_PERCENTAGE = Double.parseDouble(System.getProperty("max.timeout.percentage", "0")); + + // Other + public static final Integer PASSWORD_HASH_ITERATIONS = Integer.parseInt(System.getProperty("password.hash.iterations", "1")); + + public static boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) { + return isTimeoutPercentageWithinLimits(measurement) + && measurement.getStatistics().get(LOGIN_REQUEST_TIME).getAverage() < MAX_LOGIN_TIME_AVERAGE + && measurement.getStatistics().get(LOGOUT_REQUEST_TIME).getAverage() < MAX_LOGOUT_TIME_AVERAGE; + } + + public static boolean isTimeoutPercentageWithinLimits(PerformanceMeasurement measurement) { + boolean withinLimits = true; + for (String statistic : measurement.getStatistics().keySet()) { + withinLimits = withinLimits && measurement.getTimeoutPercentage(statistic) <= MAX_TIMEOUT_PERCENTAGE; + } + return withinLimits; + } + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoopingRunnable.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoopingRunnable.java new file mode 100644 index 0000000000..9cfcccc27b --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/LoopingRunnable.java @@ -0,0 +1,42 @@ +package org.keycloak.testsuite.performance; + +import static org.keycloak.testsuite.util.WaitUtils.pause; + +/** + * + * @author tkyjovsk + */ +public abstract class LoopingRunnable implements Runnable { + + private long sleepBetweenLoopsMillis; + private long loopCounter; + + public LoopingRunnable() { + this(0); + this.loopCounter = 0; + } + + public LoopingRunnable(long sleepBetweenLoopsMillis) { + this.sleepBetweenLoopsMillis = sleepBetweenLoopsMillis; + } + + public void setSleepBetweenLoopsMillis(long sleepBetweenLoopsMillis) { + this.sleepBetweenLoopsMillis = sleepBetweenLoopsMillis; + } + + public long getLoopCounter() { + return loopCounter; + } + + @Override + public void run() { + while (!Thread.interrupted()) { + loop(); + loopCounter++; + pause(sleepBetweenLoopsMillis); + } + } + + public abstract void loop(); + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java index deb5026f4e..c3647667d4 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/OperationTimeoutException.java @@ -6,21 +6,21 @@ package org.keycloak.testsuite.performance; */ public class OperationTimeoutException extends Exception { - private final String metric; + private final String statistic; private final long value; - public OperationTimeoutException(String metric, Throwable cause) { - this(metric, 0, cause); + public OperationTimeoutException(String statistic, Throwable cause) { + this(statistic, 0, cause); } - public OperationTimeoutException(String metric, long value, Throwable cause) { + public OperationTimeoutException(String statistic, long value, Throwable cause) { super(cause); - this.metric = metric; + this.statistic = statistic; this.value = value; } - public String getMetric() { - return metric; + public String getStatistic() { + return statistic; } public long getValue() { diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java new file mode 100644 index 0000000000..32ec4a2528 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceMeasurement.java @@ -0,0 +1,145 @@ +package org.keycloak.testsuite.performance; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import static org.keycloak.testsuite.performance.PerformanceTest.LOG; +import org.keycloak.testsuite.performance.statistics.SimpleStatistics; +import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY; + +/** + * + * @author tkyjovsk + */ +public class PerformanceMeasurement { + + private final Date started; + private final int load; + private long durationMillis; + private SimpleStatistics statistics; + private SimpleStatistics timeoutStatistics; + + public static final DateFormat ISO8601_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXX"); // should be compatible with `date --iso-8601=seconds` + public static final DateFormat RFC3339_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ssZ"); // should be compatible with `date --rfc-3339=seconds` + + public PerformanceMeasurement(int load) { + this.started = new Date(); + this.load = load; + } + + public SimpleStatistics getStatistics() { + return this.statistics; + } + + public SimpleStatistics getTimeoutStatistics() { + return this.timeoutStatistics; + } + + public void setStatistics(SimpleStatistics statistics, SimpleStatistics timeoutStatistics) { + this.durationMillis = new Date().getTime() - started.getTime(); + if (durationMillis < 0) { + throw new IllegalStateException("Cannot set a negative duration."); + } + this.statistics = statistics; + this.timeoutStatistics = timeoutStatistics; + } + + private void checkStatisticsNotNull() { + if (statistics == null || timeoutStatistics == null) { + throw new IllegalStateException("Iteration doesn't have any statistics set."); + } + } + + public double getThroughput(String statistic) { + checkStatisticsNotNull(); + return (double) statistics.get(statistic).getCount() / durationMillis * 1000; + } + + public double getTimeoutPercentage(String statistic) { + checkStatisticsNotNull(); + long timeouts = timeoutStatistics.containsKey(statistic) ? timeoutStatistics.get(statistic).getCount() : 0; + return (double) timeouts / statistics.get(statistic).getCount(); + } + + public static final Object[] HEADER = new String[]{ + "Timestamp", + "Load", + "Duration", + "Count", + "Min", + "Max", + "Average", + "Standard Deviation", + "Timeout Percentage", + "Throughput",}; + + public List toRecord(String statistic) { + checkStatisticsNotNull(); + List record = new ArrayList(); + record.add(ISO8601_DATE_FORMAT.format(started)); + record.add(load); + record.add(durationMillis); + record.add(statistics.get(statistic).getCount()); + record.add(statistics.get(statistic).getMin()); + record.add(statistics.get(statistic).getMax()); + record.add(statistics.get(statistic).getAverage()); + record.add(statistics.get(statistic).getStandardDeviation()); + record.add(getTimeoutPercentage(statistic)); + record.add(getThroughput(statistic)); + return record; + } + + public void printToCSV() { + printToCSV(null); + } + + public void printToCSV(String testName) { + checkStatisticsNotNull(); + for (String statistic : statistics.keySet()) { + + File csvFile = new File(PROJECT_BUILD_DIRECTORY + "/measurements" + (testName == null ? "" : "/" + testName), + statistic + ".csv"); + boolean csvFileCreated = false; + if (!csvFile.exists()) { + try { + csvFile.getParentFile().mkdirs(); + csvFileCreated = csvFile.createNewFile(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile, true))) { + + CSVPrinter printer = new CSVPrinter(writer, CSVFormat.RFC4180); + + if (csvFileCreated) { + printer.printRecord(HEADER); + } + printer.printRecord(toRecord(statistic)); + + printer.flush(); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + } + + public void printToLog() { + LOG.info("Measurement results:"); + LOG.info("Operation " + Arrays.toString(HEADER)); + for (String statistic : statistics.keySet()) { + LOG.info(statistic + " " + toRecord(statistic)); + } + } + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java new file mode 100644 index 0000000000..5209bbb529 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceStatistics.java @@ -0,0 +1,56 @@ +package org.keycloak.testsuite.performance; + +import java.util.concurrent.ConcurrentHashMap; +import org.keycloak.testsuite.performance.statistics.DataHoldingUpdatableStatistic; +import org.keycloak.testsuite.performance.statistics.MovingUpdatableStatistic; +import org.keycloak.testsuite.performance.statistics.SimpleStatistics; +import org.keycloak.testsuite.performance.statistics.UpdatableStatistics; +import org.keycloak.testsuite.performance.statistics.UpdatableStatistic; + +/** + * PerformanceStatistics. Concurrent hash map of UpdatableStatistic objects, + * type of which can be selected by the "statistic.type" property. + * + * @author tkyjovsk + */ +public class PerformanceStatistics extends ConcurrentHashMap implements UpdatableStatistics { + + public static final String STATISTIC_TYPE = System.getProperty("statistic.type", MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE); + + @Override + public void reset() { + clear(); + } + + private UpdatableStatistic createIfNullAndGet(String statistic) { + UpdatableStatistic updatableStatistic = get(statistic); + if (updatableStatistic == null) { + switch (STATISTIC_TYPE) { + case DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE: + updatableStatistic = new DataHoldingUpdatableStatistic(); + break; + case MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE: + updatableStatistic = new DataHoldingUpdatableStatistic(); + break; + default: + throw new IllegalStateException(String.format( + "Unknown statistic type: '%s'. Supported values: %s | %s", + STATISTIC_TYPE, + DataHoldingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE, + MovingUpdatableStatistic.STATISTIC_TYPE_PROPERTY_VALUE)); + } + put(statistic, updatableStatistic); + } + return updatableStatistic; + } + + @Override + public void addValue(String statistic, long value) { + createIfNullAndGet(statistic).addValue(value); + } + + public SimpleStatistics snapshot() { + return new SimpleStatistics(this); + } + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java index a4278e131a..24c2d90fc2 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTest.java @@ -11,8 +11,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.keycloak.testsuite.adapter.AbstractExampleAdapterTest; -import org.keycloak.testsuite.performance.metrics.impl.Results; -import org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput; import static org.keycloak.testsuite.util.WaitUtils.pause; /** @@ -21,12 +19,12 @@ import static org.keycloak.testsuite.util.WaitUtils.pause; * @author tkyjovsk */ public abstract class PerformanceTest extends AbstractExampleAdapterTest { - - private final Logger LOG = Logger.getLogger(PerformanceTest.class); - + + public static final Logger LOG = Logger.getLogger(PerformanceTest.class); + public static final Integer WARMUP_LOAD = Integer.parseInt(System.getProperty("warmup.load", "5")); public static final Integer WARMUP_DURATION = Integer.parseInt(System.getProperty("warmup.duration", "30")); - + public static final Integer INITIAL_LOAD = Integer.parseInt(System.getProperty("initial.load", "10")); // load for the first iteration public static final Integer LOAD_INCREASE = Integer.parseInt(System.getProperty("load.increase", "10")); // how many threads to add before each iteration public static final Integer LOAD_INCREASE_RATE = Integer.parseInt(System.getProperty("load.increase.rate", "2")); // how fast to add the new threads per second @@ -35,76 +33,83 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest { public static final Integer MAX_ITERATIONS = Integer.parseInt(System.getProperty("max.iterations", "10")); public static final Integer MAX_THREADS = Integer.parseInt(System.getProperty("max.threads", "1000")); - - public static final Integer SLEEP_BETWEEN_REPEATS = Integer.parseInt(System.getProperty("sleep.between.repeats", "0")); - - private final double AVERAGE_TIMEOUT_PERCENTAGE_LIMIT = Double.parseDouble(System.getProperty("average.timeout.percentage.limit", "0.01")); - + + public static final Integer SLEEP_BETWEEN_LOOPS = Integer.parseInt(System.getProperty("sleep.between.loops", "0")); + public static final Integer THREADPOOL_TERMINATION_TIMEOUT = Integer.parseInt(System.getProperty("threadpool.termination.timeout", "10")); + public static final Integer ADDITIONAL_SLEEP_AFTER = Integer.parseInt(System.getProperty("additional.sleep.after", "0")); + + public static final String SCENARIO_TIME = "SCENARIO"; + private int currentLoad; - + private ExecutorService executorService; + + protected PerformanceStatistics statistics = new PerformanceStatistics(); + protected PerformanceStatistics timeoutStatistics = new PerformanceStatistics(); // for keeping track of # of conn. timeout exceptions - protected PerformanceTestMetrics metrics = new PerformanceTestMetrics(); - protected PerformanceTestMetrics timeouts = new PerformanceTestMetrics(); - - protected List resultsList = new ArrayList<>(); - protected List timeoutResultsList = new ArrayList<>(); - + protected List measurements = new ArrayList<>(); + @Before public void before() { if (WARMUP_LOAD > INITIAL_LOAD) { throw new IllegalArgumentException("'warmup.load' cannot be larger than 'initial.load'"); } - + executorService = Executors.newFixedThreadPool(MAX_THREADS); currentLoad = 0; - - metrics.clear(); + + statistics.clear(); + timeoutStatistics.clear(); } - + @After public void after() throws IOException, InterruptedException { executorService.shutdown(); + LOG.info("Waiting for threadpool termination."); - executorService.awaitTermination(10, TimeUnit.SECONDS); + executorService.awaitTermination(THREADPOOL_TERMINATION_TIMEOUT, TimeUnit.SECONDS); + pause(ADDITIONAL_SLEEP_AFTER * 1000); + + LOG.info("Logging out all sessions."); + testRealmResource().logoutAll(); } - + @Test public void test() { - + increaseLoadBy(WARMUP_LOAD); // increase to warmup load warmup(); - + for (int i = 0; i < MAX_ITERATIONS; i++) { - + int loadIncrease = (i == 0) ? INITIAL_LOAD - WARMUP_LOAD // increase from warmup to initial load - : LOAD_INCREASE; // increase load between iterations + : LOAD_INCREASE; // increase load between measurements increaseLoadBy(loadIncrease); measurePerformance(); - + if (!isThereEnoughThreadsForNextIteration(LOAD_INCREASE)) { LOG.warn("Threadpool capacity reached. Stopping the test."); break; } - if (!isLatestResultsWithinLimits()) { - LOG.warn("The latest measurement surpassed expected limit. Stopping the test."); + if (!isLatestMeasurementWithinLimits()) { + LOG.warn("The latest measurement exceeded expected limit. Stopping the test."); break; } } - + } - + private void warmup() { LOG.info("Warming up for " + WARMUP_DURATION + " s"); pauseWithErrorChecking(WARMUP_DURATION * 1000); } - + private boolean isThereEnoughThreadsForNextIteration(int loadIncrease) { return currentLoad + loadIncrease <= MAX_THREADS; } - + private void increaseLoadBy(int loadIncrease) { if (loadIncrease < 0) { throw new IllegalArgumentException("Cannot increase load by a negative number (" + loadIncrease + ")."); @@ -113,7 +118,7 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest { throw new IllegalArgumentException("Cannot increase load beyond threadpool capacity (" + MAX_THREADS + ")."); } if (loadIncrease > 0) { - LOG.info(String.format("Increasing load from %s to %s.", currentLoad, currentLoad + loadIncrease)); + LOG.info(String.format("Increasing load from %s to %s at +%s clients/s.", currentLoad, currentLoad + loadIncrease, LOAD_INCREASE_RATE)); for (int t = 0; t < loadIncrease; t++) { executorService.submit(newRunnable()); currentLoad++; @@ -121,95 +126,98 @@ public abstract class PerformanceTest extends AbstractExampleAdapterTest { } } } - + private void measurePerformance() { - LOG.info("Measuring performance"); - LOG.info("Iteration: " + (resultsList.size() + 1)); - LOG.info("Duration: " + MEASUREMENT_DURATION + " s"); - LOG.info("Load: " + currentLoad); - - metrics.reset(); + PerformanceMeasurement measurement = new PerformanceMeasurement(currentLoad); + statistics.reset(); + timeoutStatistics.reset(); + + LOG.info(String.format("Measuring performance. Iteration: %s, Load: %s, Duration: %s s", measurements.size() + 1, currentLoad, MEASUREMENT_DURATION)); + pauseWithErrorChecking(MEASUREMENT_DURATION * 1000); - resultsList.add(metrics.computeMetrics()); - timeoutResultsList.add(timeouts.computeMetrics()); - - getLatestResults().logResults(); // to file - LOG.info("Timeouts: " + getLatestTimeoutResults()); + + measurement.setStatistics( + statistics.snapshot(), + timeoutStatistics.snapshot()); + measurements.add(measurement); + + measurement.printToCSV(getTestName()); + measurement.printToLog(); } - - protected ResultsWithThroughput getLatestResults() { - return resultsList.isEmpty() ? null : resultsList.get(resultsList.size() - 1); - } - - protected Results getLatestTimeoutResults() { - return timeoutResultsList.isEmpty() ? null : timeoutResultsList.get(timeoutResultsList.size() - 1); - } - + private Throwable error = null; - + public synchronized Throwable getError() { return error; } - + public synchronized void setError(Throwable error) { this.error = error; } - + protected void pauseWithErrorChecking(long millis) { pauseWithErrorChecking(millis, 1000); } - - protected void pauseWithErrorChecking(long millis, long checkIntervals) { - long count = millis / checkIntervals; - long remainder = millis % checkIntervals; - for (int i = 0; i < count + 1; i++) { // repeat 'count' times + once for remainder - if (i < count || remainder > 0) { // on last iteration check if any remainder - pause(checkIntervals); + + protected void pauseWithErrorChecking(long millis, long checkDurationMillis) { + long checkDurationMillisMin = Math.min(millis, checkDurationMillis); + long checkCount = millis / checkDurationMillis; + long remainder = millis % checkDurationMillis; + LOG.debug(String.format("Pause %s ms, checking errors once per %s ms", millis, checkDurationMillisMin)); + for (int i = 0; i < checkCount + 1; i++) { // loop 'count' times + once for remainder + if (i < checkCount || remainder > 0) { // on last iteration check if any remainder + pause(checkDurationMillisMin); if (getError() != null) { throw new RuntimeException("PerformanceTestRunnable threw an exception. Stopping the test.", getError()); } } } } - - protected abstract boolean isLatestResultsWithinLimits(); - - protected boolean isLatestTimeoutsWithinLimits() { - boolean socketTimeoutsWithinLimits = true; - for (String metric : getLatestResults().keySet()) { - long timemoutCount = getLatestTimeoutResults().containsKey(metric) ? getLatestTimeoutResults().get(metric).getCount() : 0; - double timeoutPercentage = (double) timemoutCount / getLatestResults().get(metric).getCount(); - socketTimeoutsWithinLimits = socketTimeoutsWithinLimits && timeoutPercentage < AVERAGE_TIMEOUT_PERCENTAGE_LIMIT; - } - return socketTimeoutsWithinLimits; + + protected PerformanceMeasurement getLatestMeasurement() { + return measurements.isEmpty() ? null : measurements.get(measurements.size() - 1); } - + + protected boolean isLatestMeasurementWithinLimits() { + return isMeasurementWithinLimits(getLatestMeasurement()); + } + + protected abstract boolean isMeasurementWithinLimits(PerformanceMeasurement measurement); + protected abstract Runnable newRunnable(); - - public abstract class Runnable extends RepeatRunnable { - - protected final Timer timer; + + public abstract class Runnable extends LoopingRunnable { + + protected final Timer timer; // for timing individual operations/requests + private final Timer scenarioTimer; // for timing the whole scenario public Runnable() { - super(SLEEP_BETWEEN_REPEATS * 1000); + super(SLEEP_BETWEEN_LOOPS * 1000); this.timer = new Timer(); + this.scenarioTimer = new Timer(); } - + @Override - public void repeat() { + public void loop() { try { + scenarioTimer.reset(); performanceScenario(); + statistics.addValue(SCENARIO_TIME, scenarioTimer.getElapsedTime()); } catch (OperationTimeoutException ex) { - timeouts.addValue(ex.getMetric(), ex.getValue()); - LOG.debug(String.format("Operatin %s timed out. Cause: %s.", ex.getMetric(), ex.getCause())); + timeoutStatistics.addValue(ex.getStatistic(), ex.getValue()); + LOG.debug(String.format("Operation %s timed out. Cause: %s.", ex.getStatistic(), ex.getCause())); } catch (AssertionError | Exception ex) { setError(ex); throw new RuntimeException(ex); } } - + public abstract void performanceScenario() throws Exception; - + } - + + public String getTestName() { + return this.getClass().getSimpleName(); + } + } diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTestMetrics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTestMetrics.java deleted file mode 100644 index 1ebaea33f2..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/PerformanceTestMetrics.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.keycloak.testsuite.performance; - -import java.util.concurrent.ConcurrentHashMap; -import org.keycloak.testsuite.performance.metrics.ComputedMetric; -import org.keycloak.testsuite.performance.metrics.ComputedMetrics; -import org.keycloak.testsuite.performance.metrics.impl.ArrayListMetric; -import org.keycloak.testsuite.performance.metrics.impl.MovingAverageMetric; -import org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput; - -/** - * - * @author tkyjovsk - */ -public class PerformanceTestMetrics extends ConcurrentHashMap implements ComputedMetrics { - - public static final String METRIC_MOVING_AVERAGE = "MovingAverage"; - public static final String METRIC_ARRAY_LIST = "ArrayList"; - public static final String METRIC = System.getProperty("metric", METRIC_MOVING_AVERAGE); - - Timer timer = new Timer(); - - @Override - public void reset() { - clear(); - timer.reset(); - } - - private ComputedMetric getOrCreate(String metric) { - ComputedMetric m = get(metric); - if (m == null) { - if (METRIC_ARRAY_LIST.equals(metric)) { - m = new ArrayListMetric(); - } else { - m = new MovingAverageMetric(); - } - put(metric, m); - } - return m; - } - - @Override - public void addValue(String metric, long value) { - getOrCreate(metric).addValue(value); - } - - @Override - public ResultsWithThroughput computeMetrics() { - return new ResultsWithThroughput(this, timer.getElapsedTime()); - } - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/RepeatRunnable.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/RepeatRunnable.java deleted file mode 100644 index f62d19608c..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/RepeatRunnable.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.keycloak.testsuite.performance; - -import static org.keycloak.testsuite.util.WaitUtils.pause; - -/** - * - * @author tkyjovsk - */ -public abstract class RepeatRunnable implements Runnable { - - private long sleepBetweenRepeatsMillis; - private long repeatCounter; - - public RepeatRunnable() { - this(0); - this.repeatCounter = 0; - } - - public RepeatRunnable(long sleepBetweenRepeatsMillis) { - this.sleepBetweenRepeatsMillis = sleepBetweenRepeatsMillis; - } - - public void setSleepBetweenRepeatsMillis(long sleepBetweenRepeatsMillis) { - this.sleepBetweenRepeatsMillis = sleepBetweenRepeatsMillis; - } - - public long getRepeatCounter() { - return repeatCounter; - } - - @Override - public void run() { - while (true) { - repeat(); - repeatCounter++; - pause(sleepBetweenRepeatsMillis); - } - } - - public abstract void repeat(); - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java index b2790985fa..de8e29f7a2 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/htmlunit/HtmlUnitLoginLogoutPerfTest.java @@ -13,18 +13,19 @@ import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.testsuite.arquillian.annotation.AppServerContainer; import org.keycloak.testsuite.performance.page.AppProfileJEE; import org.openqa.selenium.By; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME; import org.keycloak.testsuite.performance.PerformanceTest; -import static org.junit.Assert.assertTrue; import org.keycloak.testsuite.performance.OperationTimeoutException; -import static org.keycloak.testsuite.util.IOUtil.loadRealm; import org.openqa.selenium.TimeoutException; +import org.keycloak.testsuite.performance.PerformanceMeasurement; +import org.keycloak.testsuite.performance.LoginLogoutTestParameters; +import static org.junit.Assert.assertTrue; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS; +import static org.keycloak.testsuite.util.IOUtil.loadRealm; /** * @@ -65,7 +66,9 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { @Override public void addAdapterTestRealms(List testRealms) { - testRealms.add(loadRealm("/examples-realm.json")); + RealmRepresentation examplesRealm = loadRealm("/examples-realm.json"); + examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")"); + testRealms.add(examplesRealm); } @Before @@ -83,16 +86,15 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } @Override - protected boolean isLatestResultsWithinLimits() { - return getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT - && getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT; + protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) { + return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement); } public class Runnable extends HtmlUnitPerformanceTest.Runnable { @Override public void performanceScenario() throws Exception { - LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter())); + LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter())); driver.manage().deleteAllCookies(); // ACCESS @@ -104,7 +106,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } catch (TimeoutException ex) { throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex); } - metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime()); // LOGIN LOG.trace("Logging in"); @@ -119,7 +121,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } catch (TimeoutException ex) { throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex); } - metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime()); // VERIFY LOGIN LOG.trace("Verifying login"); @@ -130,7 +132,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } catch (TimeoutException ex) { throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex); } - metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime()); // LOGOUT LOG.trace("Logging out"); @@ -141,7 +143,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } catch (TimeoutException ex) { throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex); } - metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime()); // VERIFY LOGOUT LOG.trace("Verifying logout"); @@ -152,7 +154,7 @@ public class HtmlUnitLoginLogoutPerfTest extends HtmlUnitPerformanceTest { } catch (TimeoutException ex) { throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex); } - metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime()); LOG.trace("Logged out"); } diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java index 2a6e301b38..95909580f4 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientLoginLogoutPerfTest.java @@ -27,18 +27,19 @@ import org.junit.Before; import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.testsuite.arquillian.annotation.AppServerContainer; import org.keycloak.testsuite.performance.page.AppProfileJEE; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGIN_TIME_LIMIT; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.AVERAGE_LOGOUT_TIME_LIMIT; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.ACCESS_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGIN_VERIFY_REQUEST_TIME; -import static org.keycloak.testsuite.performance.LoginLogoutParameters.LOGOUT_VERIFY_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.ACCESS_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_VERIFY_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_VERIFY_REQUEST_TIME; import org.keycloak.testsuite.performance.PerformanceTest; import org.keycloak.testsuite.performance.OperationTimeoutException; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGIN_REQUEST_TIME; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.LOGOUT_REQUEST_TIME; +import org.keycloak.testsuite.performance.PerformanceMeasurement; +import org.keycloak.testsuite.performance.LoginLogoutTestParameters; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.keycloak.testsuite.performance.LoginLogoutTestParameters.PASSWORD_HASH_ITERATIONS; import static org.keycloak.testsuite.util.IOUtil.loadRealm; /** @@ -80,7 +81,9 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest { @Override public void addAdapterTestRealms(List testRealms) { - testRealms.add(loadRealm("/examples-realm.json")); + RealmRepresentation examplesRealm = loadRealm("/examples-realm.json"); + examplesRealm.setPasswordPolicy("hashIterations(" + PASSWORD_HASH_ITERATIONS + ")"); + testRealms.add(examplesRealm); } @Before @@ -98,17 +101,15 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest { } @Override - protected boolean isLatestResultsWithinLimits() { - return isLatestTimeoutsWithinLimits() - && getLatestResults().get(LOGIN_REQUEST_TIME).getAverage() < AVERAGE_LOGIN_TIME_LIMIT - && getLatestResults().get(LOGOUT_REQUEST_TIME).getAverage() < AVERAGE_LOGOUT_TIME_LIMIT; + protected boolean isMeasurementWithinLimits(PerformanceMeasurement measurement) { + return LoginLogoutTestParameters.isMeasurementWithinLimits(measurement); } public class Runnable extends HttpClientPerformanceTest.Runnable { @Override public void performanceScenario() throws IOException, OperationTimeoutException { - LOG.debug(String.format("Starting login-logout scenario #%s", getRepeatCounter())); + LOG.trace(String.format("Starting login-logout scenario #%s", getLoopCounter())); context.getCookieStore().clear(); // ACCESS @@ -118,17 +119,17 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest { LOG.trace(getSecuredPageRequest); timer.reset(); try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) { - assertEquals(HTTP_OK, r.getStatusLine().getStatusCode()); + assertEquals("ACCESS_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode()); logRedirects(); - assertEquals(1, context.getRedirectLocations().size()); - assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl)); + assertEquals("ACCESS_REQUEST has 1 redirect", 1, context.getRedirectLocations().size()); + assertTrue("ACCESS_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl)); pageContent = EntityUtils.toString(r.getEntity()); } catch (SocketException ex) { throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex); } catch (SocketTimeoutException ex) { throw new OperationTimeoutException(ACCESS_REQUEST_TIME, ex.bytesTransferred, ex); } - metrics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(ACCESS_REQUEST_TIME, timer.getElapsedTime()); // LOGIN final HttpPost loginRequest = new HttpPost(getLoginUrlFromPage(pageContent)); @@ -141,31 +142,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest { LOG.trace(loginRequest); timer.reset(); try (CloseableHttpResponse r = client.execute(loginRequest, context)) { - assertEquals(HTTP_OK, r.getStatusLine().getStatusCode()); + assertEquals("LOGIN_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode()); logRedirects(); - assertEquals(2, context.getRedirectLocations().size()); - assertTrue(getLastRedirect().toASCIIString().equals(securedUrl)); + assertEquals("LOGIN_REQUEST has 2 redirects", 2, context.getRedirectLocations().size()); + assertTrue("LOGIN_REQUEST redirects to secured page", getLastRedirect().toASCIIString().equals(securedUrl)); } catch (SocketException ex) { throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex); } catch (SocketTimeoutException ex) { throw new OperationTimeoutException(LOGIN_REQUEST_TIME, ex.bytesTransferred, ex); } - metrics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGIN_REQUEST_TIME, timer.getElapsedTime()); // VERIFY LOGIN LOG.trace("Verifying login"); LOG.trace(getSecuredPageRequest); timer.reset(); try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) { - assertEquals(HTTP_OK, r.getStatusLine().getStatusCode()); + assertEquals("LOGIN_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode()); logRedirects(); - assertEquals(0, context.getRedirectLocations().size()); + assertEquals("LOGIN_VERIFY_REQUEST has 0 redirects", 0, context.getRedirectLocations().size()); } catch (SocketException ex) { throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex); } catch (SocketTimeoutException ex) { throw new OperationTimeoutException(LOGIN_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex); } - metrics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGIN_VERIFY_REQUEST_TIME, timer.getElapsedTime()); // LOGOUT final HttpGet logoutRequest = new HttpGet(logoutUrl); @@ -173,31 +174,31 @@ public class HttpClientLoginLogoutPerfTest extends HttpClientPerformanceTest { LOG.trace(logoutRequest); timer.reset(); try (CloseableHttpResponse r = client.execute(logoutRequest, context)) { - assertEquals(HTTP_OK, r.getStatusLine().getStatusCode()); + assertEquals("LOGOUT_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode()); logRedirects(); - assertEquals(0, context.getRedirectLocations().size()); + assertEquals("LOGOUT_REQUEST has 0 redirects", 0, context.getRedirectLocations().size()); } catch (SocketException ex) { throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex); } catch (SocketTimeoutException ex) { throw new OperationTimeoutException(LOGOUT_REQUEST_TIME, ex.bytesTransferred, ex); } - metrics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGOUT_REQUEST_TIME, timer.getElapsedTime()); // VERIFY LOGOUT LOG.trace("Verifying logout"); LOG.trace(getSecuredPageRequest); timer.reset(); try (CloseableHttpResponse r = client.execute(getSecuredPageRequest, context)) { - assertEquals(HTTP_OK, r.getStatusLine().getStatusCode()); + assertEquals("LOGOUT_VERIFY_REQUEST OK", HTTP_OK, r.getStatusLine().getStatusCode()); logRedirects(); - assertEquals(1, context.getRedirectLocations().size()); - assertTrue(getLastRedirect().toASCIIString().startsWith(loginPageUrl)); + assertEquals("LOGOUT_VERIFY_REQUEST has 1 redirect", 1, context.getRedirectLocations().size()); + assertTrue("LOGOUT_VERIFY_REQUEST redirects to login page", getLastRedirect().toASCIIString().startsWith(loginPageUrl)); } catch (SocketException ex) { throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex); } catch (SocketTimeoutException ex) { throw new OperationTimeoutException(LOGOUT_VERIFY_REQUEST_TIME, ex.bytesTransferred, ex); } - metrics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime()); + statistics.addValue(LOGOUT_VERIFY_REQUEST_TIME, timer.getElapsedTime()); LOG.trace("Logged out"); diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java index d9aafafade..e6504e5bea 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/httpclient/HttpClientPerformanceTest.java @@ -12,6 +12,7 @@ import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.SocketConfig; import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.DefaultRedirectStrategy; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; @@ -45,6 +46,7 @@ public abstract class HttpClientPerformanceTest extends PerformanceTest { .setDefaultCookieStore(new BasicCookieStore()) .setDefaultRequestConfig(getDefaultRequestConfig()) .setRedirectStrategy(new CustomRedirectStrategy()) + .setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)) .build(); } diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetric.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetric.java deleted file mode 100644 index 69c063bf82..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetric.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.keycloak.testsuite.performance.metrics; - -/** - * - * @author tkyjovsk - */ -public interface ComputedMetric extends Metric { - - public void reset(); - public void addValue(long value); - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetrics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetrics.java deleted file mode 100644 index d4d9cf0721..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/ComputedMetrics.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.keycloak.testsuite.performance.metrics; - -/** - * - * @author tkyjovsk - */ -public interface ComputedMetrics extends Metrics { - - public void reset(); - public void addValue(String metric, long value); - public Metrics computeMetrics(); - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metric.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metric.java deleted file mode 100644 index b2a6809dfe..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metric.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.keycloak.testsuite.performance.metrics; - -/** - * - * @author tkyjovsk - */ -public interface Metric { - - public long getCount(); - public long getMin(); - public long getMax(); - public double getAverage(); - public double getStandardDeviation(); - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metrics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metrics.java deleted file mode 100644 index b893dc9d46..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/Metrics.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.keycloak.testsuite.performance.metrics; - -import java.util.Map; - -/** - * - * @author tkyjovsk - * @param - */ -public interface Metrics extends Map { - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ArrayListMetric.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ArrayListMetric.java deleted file mode 100644 index b6419a5ae0..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ArrayListMetric.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.keycloak.testsuite.performance.metrics.impl; - -import java.util.ArrayList; -import java.util.Collections; -import org.keycloak.testsuite.performance.metrics.ComputedMetric; - -/** - * - * @author tkyjovsk - */ -public class ArrayListMetric extends ArrayList implements ComputedMetric { - - @Override - public void reset() { - clear(); - } - - @Override - public void addValue(long value) { - add(value); - } - - @Override - public long getCount() { - return size(); - } - - @Override - public long getMin() { - return Collections.min(this); - } - - @Override - public long getMax() { - return Collections.max(this); - } - - @Override - public double getAverage() { - long sum = 0; - for (long l : this) { - sum += l; - } - return isEmpty() ? 0 : sum / size(); - } - - @Override - public double getStandardDeviation() { - double average = getAverage(); - long sumSquare = 0; - for (long l : this) { - sumSquare += l * l; - } - return isEmpty() ? 0 - : Math.sqrt(sumSquare / size() - (average * average)); - } - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Results.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Results.java deleted file mode 100644 index 8d4a53e9d7..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Results.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.keycloak.testsuite.performance.metrics.impl; - -import java.util.TreeMap; -import org.keycloak.testsuite.performance.metrics.ComputedMetrics; -import org.keycloak.testsuite.performance.metrics.Metrics; - -/** - * - * @author tkyjovsk - */ -public class Results extends TreeMap implements Metrics { - - public Results(ComputedMetrics metrics) { - for (String metric : metrics.keySet()) { - put(metric, new Result(metrics.get(metric))); - } - } - - public String getHeader() { - return "# Operation Count Min Max Average Standard-Deviation"; - } - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ResultsWithThroughput.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ResultsWithThroughput.java deleted file mode 100644 index 140de6f97a..0000000000 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/ResultsWithThroughput.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.keycloak.testsuite.performance.metrics.impl; - -import org.jboss.logging.Logger; -import java.util.HashMap; -import java.util.Map; -import org.keycloak.testsuite.performance.metrics.ComputedMetrics; - -/** - * - * @author tkyjovsk - */ -public class ResultsWithThroughput extends Results { - - public static final Logger LOG = Logger.getLogger(ResultsWithThroughput.class); - private final Map throughput; - - public ResultsWithThroughput(ComputedMetrics metrics, long durationMillis) { - super(metrics); - throughput = new HashMap<>(); - for (String metric : keySet()) { - throughput.put(metric, (double) get(metric).getCount() / durationMillis * 1000); - } - } - - public Map getThroughput() { - return throughput; - } - - @Override - public String toString() { - return "Results: " + super.toString() + "\n" - + "Throughput: " + getThroughput(); - } - - @Override - public String getHeader() { - return super.getHeader() + " Throughput"; - - } - - public void logResults() { - LOG.info(getHeader()); - for (String metric : keySet()) { - LOG.info(metric + " " + get(metric).toLogString() + " " + getThroughput().get(metric)); - } - } - -} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/DataHoldingUpdatableStatistic.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/DataHoldingUpdatableStatistic.java new file mode 100644 index 0000000000..91227492b2 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/DataHoldingUpdatableStatistic.java @@ -0,0 +1,62 @@ +package org.keycloak.testsuite.performance.statistics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * + * @author tkyjovsk + */ +public class DataHoldingUpdatableStatistic implements UpdatableStatistic { + + public static final String STATISTIC_TYPE_PROPERTY_VALUE = "data"; + + private final List data = Collections.synchronizedList(new ArrayList()); + + @Override + public synchronized void reset() { + data.clear(); + } + + @Override + public synchronized void addValue(long value) { + data.add(value); + } + + @Override + public synchronized long getCount() { + return data.size(); + } + + @Override + public synchronized long getMin() { + return Collections.min(data); + } + + @Override + public synchronized long getMax() { + return Collections.max(data); + } + + @Override + public synchronized double getAverage() { + long sum = 0; + for (long l : data) { + sum += l; + } + return data.isEmpty() ? 0 : sum / data.size(); + } + + @Override + public synchronized double getStandardDeviation() { + double average = getAverage(); + long sumSquare = 0; + for (long l : data) { + sumSquare += l * l; + } + return data.isEmpty() ? 0 + : Math.sqrt(sumSquare / data.size() - (average * average)); + } + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/MovingAverageMetric.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/MovingUpdatableStatistic.java similarity index 74% rename from testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/MovingAverageMetric.java rename to testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/MovingUpdatableStatistic.java index 7291db453d..553a80cfb8 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/MovingAverageMetric.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/MovingUpdatableStatistic.java @@ -1,13 +1,15 @@ -package org.keycloak.testsuite.performance.metrics.impl; +package org.keycloak.testsuite.performance.statistics; import java.math.BigDecimal; -import org.keycloak.testsuite.performance.metrics.ComputedMetric; /** + * Allows to compute statistical values without holding the actual measurements. * * @author tkyjovsk */ -public final class MovingAverageMetric implements ComputedMetric { +public final class MovingUpdatableStatistic implements UpdatableStatistic { + + public static final String STATISTIC_TYPE_PROPERTY_VALUE = "moving"; private BigDecimal sum; private BigDecimal sumSquare; @@ -15,7 +17,7 @@ public final class MovingAverageMetric implements ComputedMetric { private long min; private long max; - public MovingAverageMetric() { + public MovingUpdatableStatistic() { reset(); } @@ -29,17 +31,26 @@ public final class MovingAverageMetric implements ComputedMetric { } @Override - public long getCount() { + public synchronized void addValue(long value) { + sum = sum.add(new BigDecimal(value)); + sumSquare = sumSquare.add(new BigDecimal(value * value)); + min = Math.min(min, value); + max = Math.max(max, value); + count++; + } + + @Override + public synchronized long getCount() { return count; } @Override - public long getMin() { + public synchronized long getMin() { return min; } @Override - public long getMax() { + public synchronized long getMax() { return max; } @@ -56,15 +67,6 @@ public final class MovingAverageMetric implements ComputedMetric { : Math.sqrt(sumSquare.longValue() / count - (average * average)); } - @Override - public synchronized void addValue(long value) { - sum = sum.add(new BigDecimal(value)); - sumSquare = sumSquare.add(new BigDecimal(value * value)); - min = Math.min(min, value); - max = Math.max(max, value); - count++; - } - @Override public String toString() { return Double.toString(getAverage()); diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Result.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistic.java similarity index 69% rename from testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Result.java rename to testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistic.java index 35d0c95793..a142882ba1 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/metrics/impl/Result.java +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistic.java @@ -1,14 +1,14 @@ -package org.keycloak.testsuite.performance.metrics.impl; +package org.keycloak.testsuite.performance.statistics; import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.commons.lang.builder.ToStringStyle; -import org.keycloak.testsuite.performance.metrics.Metric; /** + * SimpleStatistic just holds data. * * @author tkyjovsk */ -public class Result implements Metric { +public class SimpleStatistic implements Statistic { private final long count; private final long min; @@ -16,7 +16,7 @@ public class Result implements Metric { private final double average; private final double standardDeviation; - public Result(long count, long min, long max, double average, double standardDeviation) { + public SimpleStatistic(long count, long min, long max, double average, double standardDeviation) { this.count = count; this.min = min; this.max = max; @@ -24,13 +24,13 @@ public class Result implements Metric { this.standardDeviation = standardDeviation; } - public Result(Metric metric) { + public SimpleStatistic(Statistic statistic) { this( - metric.getCount(), - metric.getMin(), - metric.getMax(), - metric.getAverage(), - metric.getStandardDeviation()); + statistic.getCount(), + statistic.getMin(), + statistic.getMax(), + statistic.getAverage(), + statistic.getStandardDeviation()); } @Override diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java new file mode 100644 index 0000000000..185e0c615c --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/SimpleStatistics.java @@ -0,0 +1,17 @@ +package org.keycloak.testsuite.performance.statistics; + +import java.util.TreeMap; + +/** + * + * @author tkyjovsk + */ +public class SimpleStatistics extends TreeMap implements Statistics { + + public SimpleStatistics(Statistics statistics) { + for (Object statistic : statistics.keySet()) { + put(statistic.toString(), new SimpleStatistic((Statistic) statistics.get(statistic))); + } + } + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistic.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistic.java new file mode 100644 index 0000000000..44e526243c --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistic.java @@ -0,0 +1,17 @@ +package org.keycloak.testsuite.performance.statistics; + +/** + * Statistic provides statistical information about a data set. + * Number of measurements, minimum/maximum/average value and standard deviation. + * + * @author tkyjovsk + */ +public interface Statistic { + + public long getCount(); + public long getMin(); + public long getMax(); + public double getAverage(); + public double getStandardDeviation(); + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistics.java new file mode 100644 index 0000000000..e17bf38dac --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/Statistics.java @@ -0,0 +1,13 @@ +package org.keycloak.testsuite.performance.statistics; + +import java.util.Map; + +/** + * A Map of named statistics. + * + * @author tkyjovsk + * @param + */ +public interface Statistics extends Map { + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistic.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistic.java new file mode 100644 index 0000000000..28d88a1e01 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistic.java @@ -0,0 +1,15 @@ +package org.keycloak.testsuite.performance.statistics; + +/** + * UpdatableStatistic. A Statistic that can be updated, e.g. from PerformanceTest.Runnable. + * Implementations should be thread-safe. + * + * @author tkyjovsk + */ +public interface UpdatableStatistic extends Statistic { + + public void reset(); + + public void addValue(long value); + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistics.java b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistics.java new file mode 100644 index 0000000000..1577adb36b --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/java/org/keycloak/testsuite/performance/statistics/UpdatableStatistics.java @@ -0,0 +1,13 @@ +package org.keycloak.testsuite.performance.statistics; + +/** + * + * @author tkyjovsk + */ +public interface UpdatableStatistics extends Statistics { + + public void reset(); + + public void addValue(String statistic, long value); + +} diff --git a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties index 7c833eee85..b931f7373d 100644 --- a/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties +++ b/testsuite/integration-arquillian/tests/other/adapters/jboss/remote/src/test/resources/log4j.properties @@ -8,6 +8,7 @@ log4j.appender.DEFAULT.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p [%c] [%t] log4j.logger.org.keycloak=OFF log4j.logger.org.keycloak.testsuite=INFO +log4j.logger.org.keycloak.testsuite.performance.PerformanceTest=INFO # HtmlUnit log4j.logger.org.keycloak.testsuite.performance.htmlunit.HtmlUnitLoginLogoutPerfTest=${logging.loginlogout} @@ -17,14 +18,3 @@ log4j.logger.com.gargoylesoftware.htmlunit=OFF log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientLoginLogoutPerfTest=${logging.loginlogout} log4j.logger.org.keycloak.testsuite.performance.httpclient.HttpClientPerformanceTest$CustomRedirectStrategy=OFF - -# RESULTS - -log4j.appender.RESULTS=org.apache.log4j.FileAppender -log4j.appender.RESULTS.file=target/results.log -log4j.appender.RESULTS.immediateFlush=true -log4j.appender.RESULTS.append=true -log4j.appender.RESULTS.layout=org.apache.log4j.PatternLayout -log4j.appender.RESULTS.layout.ConversionPattern=%d{HHmmss} %m%n - -log4j.logger.org.keycloak.testsuite.performance.metrics.impl.ResultsWithThroughput=INFO, RESULTS