diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java index 673bc7c4fd..fc63b2291b 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java @@ -17,7 +17,9 @@ package org.keycloak.testsuite.arquillian; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * @@ -33,6 +35,8 @@ public final class TestContext { private final List appServerBackendsInfo = new ArrayList<>(); private boolean adminLoggedIn; + + private final Map customContext = new HashMap<>(); public TestContext(SuiteContext suiteContext, Class testClass) { this.suiteContext = suiteContext; @@ -88,4 +92,12 @@ public final class TestContext { + (isAdapterTest() ? "App server container: " + getAppServerInfo() + "\n" : ""); } + public Object getCustomValue(Object key) { + return customContext.get(key); + } + + public void setCustomValue(Object key, Object value) { + customContext.put(key, value); + } + } diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/IOUtil.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/IOUtil.java index 23103d3339..d8729149ac 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/IOUtil.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/IOUtil.java @@ -43,6 +43,8 @@ public class IOUtil { private static final Logger log = Logger.getLogger(IOUtil.class); + public static final File PROJECT_BUILD_DIRECTORY = new File(System.getProperty("project.build.directory", "target")); + public static T loadJson(InputStream is, Class type) { try { return JsonSerialization.readValue(is, type); diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/Timer.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/Timer.java index 1b2ec652b1..082a2a400b 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/Timer.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/Timer.java @@ -14,15 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.keycloak.testsuite.util; -import java.text.MessageFormat; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; +import org.apache.commons.io.IOUtils; +import org.jboss.logging.Logger; +import org.jfree.chart.ChartFactory; +import org.jfree.chart.ChartUtilities; +import org.jfree.chart.JFreeChart; +import org.jfree.chart.plot.PlotOrientation; +import org.jfree.data.xy.XYSeries; +import org.jfree.data.xy.XYSeriesCollection; +import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY; +import static org.jgroups.util.Util.assertTrue; /** * @@ -30,45 +43,129 @@ import java.util.Map; */ public class Timer { - private static Long time; + public static final Timer DEFAULT = new Timer(); - private static final Map> stats = new HashMap<>(); + protected final Logger log = Logger.getLogger(Timer.class); - public static void time() { - time = new Date().getTime(); + protected static final File DATA_DIR = new File(PROJECT_BUILD_DIRECTORY, "stats/data"); + protected static final File CHARTS_DIR = new File(PROJECT_BUILD_DIRECTORY, "stats/charts"); + + public static final String DEFAULT_OPERATION = "DEFAULT_OPERATION"; + + private Long time; + private String operation = DEFAULT_OPERATION; + private final Map> stats = new TreeMap<>(); + + public long elapsedTime() { + long elapsedTime = 0; + if (time == null) { + } else { + elapsedTime = new Date().getTime() - time; + } + return elapsedTime; } - public static void time(String operation) { - if (time == null) { - System.out.println(MessageFormat.format("Starting timer for operation {0}", operation)); - time(); - } else { - long timeOrig = time; - time(); - logOperation(operation, time - timeOrig); - System.out.println(MessageFormat.format("Operation {0} took {1} ms", operation, time - timeOrig)); + public void reset() { + reset(operation); // log last operation + } + + public void reset(String operation) { + reset(operation, true); + } + + public void reset(String newOperation, boolean logOperationOnChange) { + if (time != null) { + if (operation.equals(newOperation) || logOperationOnChange) { + logOperation(operation, elapsedTime()); + } + } + time = new Date().getTime(); + if (!operation.equals(newOperation)) { + operation = newOperation; + log.info(String.format("Operation '%s' started.", newOperation)); } } - private static void logOperation(String operation, long delta) { + private void logOperation(String operation, long duration) { if (!stats.containsKey(operation)) { stats.put(operation, new ArrayList()); } - stats.get(operation).add(delta); + stats.get(operation).add(duration); + log.info(String.format("Operation '%s' took: %s ms", operation, duration)); } - public static void printStats() { - if (!stats.isEmpty()) { - System.out.println("OPERATION STATS:"); - } - for (String op : stats.keySet()) { - long sum = 0; - for (Long t : stats.get(op)) { - sum += t; + public void clearStats() { + clearStats(true, true, true); + } + + public void clearStats(boolean logStats, boolean saveData, boolean saveCharts) { + if (logStats) { + log.info("Timer Statistics:"); + for (String op : stats.keySet()) { + long sum = 0; + for (Long duration : stats.get(op)) { + sum += duration; + } + log.info(String.format("Operation '%s' average: %s ms", op, sum / stats.get(op).size())); + } + } + if (PROJECT_BUILD_DIRECTORY.exists()) { + DATA_DIR.mkdirs(); + CHARTS_DIR.mkdirs(); + for (String op : stats.keySet()) { + if (saveData) { + saveData(op); + } + if (saveCharts) { + saveChart(op); + } } - System.out.println(MessageFormat.format("Operation {0} average time: {1,number,#} ms", op, sum / stats.get(op).size())); } stats.clear(); } + private void saveData(String op) { + try { + File f = new File(DATA_DIR, op.replace(" ", "_") + ".txt"); + if (!f.createNewFile()) { + throw new IOException("Couldn't create file: " + f); + } + OutputStream stream = new BufferedOutputStream(new FileOutputStream(f)); + for (Long duration : stats.get(op)) { + IOUtils.write(duration.toString(), stream); + IOUtils.write("\n", stream); + } + stream.flush(); + IOUtils.closeQuietly(stream); + } catch (IOException ex) { + log.error("Unable to save data for operation '" + op + "'", ex); + } + } + + private void saveChart(String op) { + XYSeries series = new XYSeries(op); + int i = 0; + for (Long duration : stats.get(op)) { + series.add(++i, duration); + } + final XYSeriesCollection data = new XYSeriesCollection(series); + final JFreeChart chart = ChartFactory.createXYLineChart( + op, + "Operations", + "Duration (ms)", + data, + PlotOrientation.VERTICAL, + true, + true, + false + ); + try { + ChartUtilities.saveChartAsPNG( + new File(CHARTS_DIR, op.replace(" ", "_") + ".png"), + chart, 640, 480); + } catch (IOException ex) { + log.warn("Unable to save chart for operation '" + op + "'."); + } + } + } diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java index 92dc9c57e2..1e5145cc79 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java @@ -16,6 +16,7 @@ */ package org.keycloak.testsuite; +import java.io.File; import org.keycloak.testsuite.arquillian.TestContext; import java.util.ArrayList; import java.util.List; @@ -36,7 +37,6 @@ import org.keycloak.admin.client.resource.RealmsResource; import org.keycloak.models.Constants; import org.keycloak.representations.idm.RealmRepresentation; import org.keycloak.representations.idm.UserRepresentation; -import static org.keycloak.testsuite.admin.Users.setPasswordFor; import org.keycloak.testsuite.arquillian.AuthServerTestEnricher; import org.keycloak.testsuite.arquillian.SuiteContext; import org.keycloak.testsuite.auth.page.WelcomePage; @@ -52,6 +52,8 @@ import org.keycloak.testsuite.auth.page.login.OIDCLogin; import org.keycloak.testsuite.auth.page.login.UpdatePassword; import org.keycloak.testsuite.util.Timer; import org.keycloak.testsuite.util.WaitUtils; +import static org.keycloak.testsuite.admin.Users.setPasswordFor; +import static org.keycloak.testsuite.admin.Users.setPasswordFor; /** * @@ -124,7 +126,6 @@ public abstract class AbstractKeycloakTest { public void afterAbstractKeycloakTest() { // removeTestRealms(); // keeping test realms after test to be able to inspect failures, instead deleting existing realms before import // keycloak.close(); // keeping admin connection open - Timer.printStats(); } private void updateMasterAdminPassword() { diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/account/ResetCredentialsTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/account/ResetCredentialsTest.java old mode 100755 new mode 100644 diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/servlet/AbstractDemoServletsAdapterTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/servlet/AbstractDemoServletsAdapterTest.java old mode 100755 new mode 100644 diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/user/AbstractUserTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/user/AbstractUserTest.java new file mode 100644 index 0000000000..d7dbbdc328 --- /dev/null +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/user/AbstractUserTest.java @@ -0,0 +1,52 @@ +package org.keycloak.testsuite.user; + +import javax.ws.rs.core.Response; +import static javax.ws.rs.core.Response.Status.CREATED; +import org.keycloak.admin.client.resource.UserResource; +import org.keycloak.admin.client.resource.UsersResource; +import org.keycloak.representations.idm.UserRepresentation; +import static org.keycloak.testsuite.admin.ApiUtil.getCreatedId; +import static org.junit.Assert.assertEquals; +import org.keycloak.testsuite.AbstractAuthTest; + +/** + * + * @author tkyjovsk + */ +public abstract class AbstractUserTest extends AbstractAuthTest { + + protected UsersResource users() { + return testRealmResource().users(); + } + + protected UserResource user(UserRepresentation user) { + if (user.getId()==null) { + throw new IllegalStateException("User id cannot be null."); + } + return user(user.getId()); + } + + protected UserResource user(String id) { + return users().get(id); + } + + public static UserRepresentation createUserRep(String username) { + UserRepresentation user = new UserRepresentation(); + user.setUsername(username); + user.setEmail(username + "@email.test"); + return user; + } + + public UserRepresentation createUser(UserRepresentation user) { + return createUser(users(), user); + } + + public UserRepresentation createUser(UsersResource users, UserRepresentation user) { + Response response = users.create(user); + assertEquals(CREATED.getStatusCode(), response.getStatus()); + user.setId(getCreatedId(response)); + response.close(); + return user; + } + +} diff --git a/testsuite/integration-arquillian/tests/other/jpa-performance/README.md b/testsuite/integration-arquillian/tests/other/jpa-performance/README.md new file mode 100644 index 0000000000..d38961620e --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/jpa-performance/README.md @@ -0,0 +1,48 @@ +# Keycloak JPA Performance Tests + +## How to run + +1. Build the Arquilian Base Testsuite module: `/testsuite/integration-arquillian/base` +2. Run the test from this module using `mvn test` or `mvn clean test`. + +Optional parameters: +``` +-Dmany.users.count=10000 +-Dmany.users.batch=1000 +``` + +### With MySQL + +Start dockerized MySQL: +``` +docker run --name mysql-keycloak -e MYSQL_ROOT_PASSWORD=keycloak -e MYSQL_DATABASE=keycloak -e MYSQL_USER=keycloak -e MYSQL_PASSWORD=keycloak -d -p 3306:3306 mysql +``` + +Additional test parameters: +``` +-Pclean-jpa +-Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak +-Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver +-Dkeycloak.connectionsJpa.user=keycloak +-Dkeycloak.connectionsJpa.password=keycloak +``` + +### With PostgreSQL + +Start dockerized PostgreSQL: +``` +docker run --name postgres-keycloak -e POSTGRES_PASSWORD=keycloak -d -p 5432:5432 postgres +``` + +Additional test parameters: +``` +-Pclean-jpa +-Dkeycloak.connectionsJpa.url=jdbc:postgresql://localhost/postgres +-Dkeycloak.connectionsJpa.driver=org.postgresql.Driver +-Dkeycloak.connectionsJpa.user=postgres +-Dkeycloak.connectionsJpa.password=keycloak +``` + +## Reports + +Test creates reports in `target/stats`. diff --git a/testsuite/integration-arquillian/tests/other/jpa-performance/pom.xml b/testsuite/integration-arquillian/tests/other/jpa-performance/pom.xml new file mode 100644 index 0000000000..11ec6e7a45 --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/jpa-performance/pom.xml @@ -0,0 +1,34 @@ + + + + + + 4.0.0 + + + org.keycloak.testsuite + integration-arquillian-tests-other + 1.9.1.Final-SNAPSHOT + + + integration-arquillian-tests-jpa-performance + + Keycloak JPA Performance Tests + + diff --git a/testsuite/integration-arquillian/tests/other/jpa-performance/src/test/java/org/keycloak/testsuite/user/ManyUsersTest.java b/testsuite/integration-arquillian/tests/other/jpa-performance/src/test/java/org/keycloak/testsuite/user/ManyUsersTest.java new file mode 100644 index 0000000000..4916393bff --- /dev/null +++ b/testsuite/integration-arquillian/tests/other/jpa-performance/src/test/java/org/keycloak/testsuite/user/ManyUsersTest.java @@ -0,0 +1,119 @@ +package org.keycloak.testsuite.user; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.keycloak.representations.idm.UserRepresentation; +import org.keycloak.testsuite.util.Timer; +import org.keycloak.representations.idm.RealmRepresentation; +import org.keycloak.util.JsonSerialization; +import static org.junit.Assert.fail; +import org.keycloak.admin.client.resource.RealmResource; +import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY; + +/** + * + * @author tkyjovsk + */ +public class ManyUsersTest extends AbstractUserTest { + + private static final int COUNT = Integer.parseInt(System.getProperty("many.users.count", "10000")); + private static final int BATCH = Integer.parseInt(System.getProperty("many.users.batch", "1000")); + + private static final String REALM = "realm_with_many_users"; + + private List users; + + private final Timer realmTimer = new Timer(); + private final Timer usersTimer = new Timer(); + + protected RealmResource realmResource() { + return realmsResouce().realm(REALM); + } + + @Before + public void before() { + users = new LinkedList<>(); + for (int i = 0; i < COUNT; i++) { + users.add(createUserRep("user" + i)); + } + + realmTimer.reset("create realm before test"); + RealmRepresentation realm = new RealmRepresentation(); + realm.setRealm(REALM); + realmsResouce().create(realm); + } + + @After + public void after() { + realmTimer.clearStats(true, true, false); + usersTimer.clearStats(); + } + + @Test + public void manyUsers() throws IOException { + RealmRepresentation realm = realmResource().toRepresentation(); + realm.setUsers(users); + + // CREATE + realmTimer.reset("create " + users.size() + " users"); + usersTimer.reset("create " + BATCH + " users"); + int i = 0; + for (UserRepresentation user : users) { + createUser(realmResource().users(), user); + if (++i % BATCH == 0) { + usersTimer.reset(); + log.info("Created users: " + i + " / " + users.size()); + } + } + if (i % BATCH != 0) { + usersTimer.reset(); + log.info("Created users: " + i + " / " + users.size()); + } + + // SAVE REALM + realmTimer.reset("save realm with " + users.size() + " users"); + File realmFile = new File(PROJECT_BUILD_DIRECTORY, REALM + ".json"); + JsonSerialization.writeValueToStream(new BufferedOutputStream(new FileOutputStream(realmFile)), realm); + + // DELETE REALM + realmTimer.reset("delete realm with " + users.size() + " users"); + realmResource().remove(); + try { + realmResource().toRepresentation(); + fail("realm not deleted"); + } catch (Exception ex) { + log.debug("realm deleted"); + } + + // RE-IMPORT SAVED REALM + realmTimer.reset("re-import realm with " + realm.getUsers().size() + " users"); + realmsResouce().create(realm); + realmTimer.reset("load " + realm.getUsers().size() + " users"); + users = realmResource().users().search("", 0, Integer.MAX_VALUE); + + // DELETE INDIVIDUAL USERS + realmTimer.reset("delete " + users.size() + " users"); + usersTimer.reset("delete " + BATCH + " users", false); + i = 0; + for (UserRepresentation user : users) { + realmResource().users().get(user.getId()).remove(); + if (++i % BATCH == 0) { + usersTimer.reset(); + log.info("Deleted users: " + i + " / " + users.size()); + } + } + if (i % BATCH != 0) { + usersTimer.reset(); + log.info("Deleted users: " + i + " / " + users.size()); + } + realmTimer.reset(); + } + +} diff --git a/testsuite/integration-arquillian/tests/other/pom.xml b/testsuite/integration-arquillian/tests/other/pom.xml index 8b04c0777d..ec894b44a9 100644 --- a/testsuite/integration-arquillian/tests/other/pom.xml +++ b/testsuite/integration-arquillian/tests/other/pom.xml @@ -146,6 +146,12 @@ mod_auth_mellon + + jpa-performance + + jpa-performance + + diff --git a/testsuite/integration-arquillian/tests/pom.xml b/testsuite/integration-arquillian/tests/pom.xml index b2438dd4aa..7addcc0f05 100644 --- a/testsuite/integration-arquillian/tests/pom.xml +++ b/testsuite/integration-arquillian/tests/pom.xml @@ -108,6 +108,7 @@ maven-surefire-plugin + ${project.build.directory} ${browser} ${firefox_binary} false @@ -223,6 +224,12 @@ 2.1.0.Alpha3 + + jfree + jfreechart + 1.0.13 + + + + + + keycloak.connectionsJpa.driver + com.mysql.jdbc.Driver + + + mysql + + + mysql + mysql-connector-java + + + + + + + + + keycloak.connectionsJpa.driver + org.postgresql.Driver + + + postgresql + + + org.postgresql + postgresql + ${postgresql.version} + + + + + + clean-jpa + + + + + org.liquibase + liquibase-maven-plugin + + META-INF/jpa-changelog-master.xml + + ${keycloak.connectionsJpa.url} + ${keycloak.connectionsJpa.driver} + ${keycloak.connectionsJpa.user} + ${keycloak.connectionsJpa.password} + + false + ${keycloak.connectionsJpa.liquibaseDatabaseClass} + + + + clean-jpa + clean + + dropAll + + + + + + + + + auth-server-wildfly