Merge pull request #2282 from tkyjovsk/jpa-performance

Added some jpa perf tests (adding/removing many users)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java

@@ -17,7 +17,9 @@
 package org.keycloak.testsuite.arquillian;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  *
@@ -33,6 +35,8 @@ public final class TestContext {
     private final List<ContainerInfo> appServerBackendsInfo = new ArrayList<>();
 
     private boolean adminLoggedIn;
+    
+    private final Map customContext = new HashMap<>();
 
     public TestContext(SuiteContext suiteContext, Class testClass) {
         this.suiteContext = suiteContext;
@@ -88,4 +92,12 @@ public final class TestContext {
                 + (isAdapterTest() ? "App server container: " + getAppServerInfo() + "\n" : "");
     }
 
+    public Object getCustomValue(Object key) {
+        return customContext.get(key);
+    }
+    
+    public void setCustomValue(Object key, Object value) {
+        customContext.put(key, value);
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/IOUtil.java

@@ -43,6 +43,8 @@ public class IOUtil {
 
     private static final Logger log = Logger.getLogger(IOUtil.class);
 
+    public static final File PROJECT_BUILD_DIRECTORY = new File(System.getProperty("project.build.directory", "target"));
+
     public static <T> T loadJson(InputStream is, Class<T> type) {
         try {
             return JsonSerialization.readValue(is, type);

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/Timer.java

@@ -14,15 +14,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.keycloak.testsuite.util;
 
-import java.text.MessageFormat;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
+import org.apache.commons.io.IOUtils;
+import org.jboss.logging.Logger;
+import org.jfree.chart.ChartFactory;
+import org.jfree.chart.ChartUtilities;
+import org.jfree.chart.JFreeChart;
+import org.jfree.chart.plot.PlotOrientation;
+import org.jfree.data.xy.XYSeries;
+import org.jfree.data.xy.XYSeriesCollection;
+import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
+import static org.jgroups.util.Util.assertTrue;
 
 /**
  *
@@ -30,45 +43,129 @@ import java.util.Map;
  */
 public class Timer {
 
-    private static Long time;
+    public static final Timer DEFAULT = new Timer();
 
-    private static final Map<String, List<Long>> stats = new HashMap<>();
+    protected final Logger log = Logger.getLogger(Timer.class);
 
-    public static void time() {
-        time = new Date().getTime();
+    protected static final File DATA_DIR = new File(PROJECT_BUILD_DIRECTORY, "stats/data");
+    protected static final File CHARTS_DIR = new File(PROJECT_BUILD_DIRECTORY, "stats/charts");
+
+    public static final String DEFAULT_OPERATION = "DEFAULT_OPERATION";
+
+    private Long time;
+    private String operation = DEFAULT_OPERATION;
+    private final Map<String, List<Long>> stats = new TreeMap<>();
+
+    public long elapsedTime() {
+        long elapsedTime = 0;
+        if (time == null) {
+        } else {
+            elapsedTime = new Date().getTime() - time;
+        }
+        return elapsedTime;
     }
 
-    public static void time(String operation) {
-        if (time == null) {
-            System.out.println(MessageFormat.format("Starting timer for operation {0}", operation));
-            time();
-        } else {
-            long timeOrig = time;
-            time();
-            logOperation(operation, time - timeOrig);
-            System.out.println(MessageFormat.format("Operation {0} took {1} ms", operation, time - timeOrig));
+    public void reset() {
+        reset(operation); // log last operation
+    }
+
+    public void reset(String operation) {
+        reset(operation, true);
+    }
+
+    public void reset(String newOperation, boolean logOperationOnChange) {
+        if (time != null) {
+            if (operation.equals(newOperation) || logOperationOnChange) {
+                logOperation(operation, elapsedTime());
+            }
+        }
+        time = new Date().getTime();
+        if (!operation.equals(newOperation)) {
+            operation = newOperation;
+            log.info(String.format("Operation '%s' started.", newOperation));
         }
     }
 
-    private static void logOperation(String operation, long delta) {
+    private void logOperation(String operation, long duration) {
         if (!stats.containsKey(operation)) {
             stats.put(operation, new ArrayList<Long>());
         }
-        stats.get(operation).add(delta);
+        stats.get(operation).add(duration);
+        log.info(String.format("Operation '%s' took: %s ms", operation, duration));
     }
 
-    public static void printStats() {
-        if (!stats.isEmpty()) {
-            System.out.println("OPERATION STATS:");
-        }
-        for (String op : stats.keySet()) {
-            long sum = 0;
-            for (Long t : stats.get(op)) {
-                sum += t;
+    public void clearStats() {
+        clearStats(true, true, true);
+    }
+
+    public void clearStats(boolean logStats, boolean saveData, boolean saveCharts) {
+        if (logStats) {
+            log.info("Timer Statistics:");
+            for (String op : stats.keySet()) {
+                long sum = 0;
+                for (Long duration : stats.get(op)) {
+                    sum += duration;
+                }
+                log.info(String.format("Operation '%s' average: %s ms", op, sum / stats.get(op).size()));
+            }
+        }
+        if (PROJECT_BUILD_DIRECTORY.exists()) {
+            DATA_DIR.mkdirs();
+            CHARTS_DIR.mkdirs();
+            for (String op : stats.keySet()) {
+                if (saveData) {
+                    saveData(op);
+                }
+                if (saveCharts) {
+                    saveChart(op);
+                }
             }
-            System.out.println(MessageFormat.format("Operation {0} average time: {1,number,#} ms", op, sum / stats.get(op).size()));
         }
         stats.clear();
     }
 
+    private void saveData(String op) {
+        try {
+            File f = new File(DATA_DIR, op.replace(" ", "_") + ".txt");
+            if (!f.createNewFile()) {
+                throw new IOException("Couldn't create file: " + f);
+            }
+            OutputStream stream = new BufferedOutputStream(new FileOutputStream(f));
+            for (Long duration : stats.get(op)) {
+                IOUtils.write(duration.toString(), stream);
+                IOUtils.write("\n", stream);
+            }
+            stream.flush();
+            IOUtils.closeQuietly(stream);
+        } catch (IOException ex) {
+            log.error("Unable to save data for operation '" + op + "'", ex);
+        }
+    }
+
+    private void saveChart(String op) {
+        XYSeries series = new XYSeries(op);
+        int i = 0;
+        for (Long duration : stats.get(op)) {
+            series.add(++i, duration);
+        }
+        final XYSeriesCollection data = new XYSeriesCollection(series);
+        final JFreeChart chart = ChartFactory.createXYLineChart(
+                op,
+                "Operations",
+                "Duration (ms)",
+                data,
+                PlotOrientation.VERTICAL,
+                true,
+                true,
+                false
+        );
+        try {
+            ChartUtilities.saveChartAsPNG(
+                    new File(CHARTS_DIR, op.replace(" ", "_") + ".png"),
+                    chart, 640, 480);
+        } catch (IOException ex) {
+            log.warn("Unable to save chart for operation '" + op + "'.");
+        }
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java

@@ -16,6 +16,7 @@
  */
 package org.keycloak.testsuite;
 
+import java.io.File;
 import org.keycloak.testsuite.arquillian.TestContext;
 import java.util.ArrayList;
 import java.util.List;
@@ -36,7 +37,6 @@ import org.keycloak.admin.client.resource.RealmsResource;
 import org.keycloak.models.Constants;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
-import static org.keycloak.testsuite.admin.Users.setPasswordFor;
 import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
 import org.keycloak.testsuite.arquillian.SuiteContext;
 import org.keycloak.testsuite.auth.page.WelcomePage;
@@ -52,6 +52,8 @@ import org.keycloak.testsuite.auth.page.login.OIDCLogin;
 import org.keycloak.testsuite.auth.page.login.UpdatePassword;
 import org.keycloak.testsuite.util.Timer;
 import org.keycloak.testsuite.util.WaitUtils;
+import static org.keycloak.testsuite.admin.Users.setPasswordFor;
+import static org.keycloak.testsuite.admin.Users.setPasswordFor;
 
 /**
  *
@@ -124,7 +126,6 @@ public abstract class AbstractKeycloakTest {
     public void afterAbstractKeycloakTest() {
 //        removeTestRealms(); // keeping test realms after test to be able to inspect failures, instead deleting existing realms before import
 //        keycloak.close(); // keeping admin connection open
-        Timer.printStats();
     }
 
     private void updateMasterAdminPassword() {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/user/AbstractUserTest.java

@@ -0,0 +1,52 @@
+package org.keycloak.testsuite.user;
+
+import javax.ws.rs.core.Response;
+import static javax.ws.rs.core.Response.Status.CREATED;
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.admin.client.resource.UsersResource;
+import org.keycloak.representations.idm.UserRepresentation;
+import static org.keycloak.testsuite.admin.ApiUtil.getCreatedId;
+import static org.junit.Assert.assertEquals;
+import org.keycloak.testsuite.AbstractAuthTest;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractUserTest extends AbstractAuthTest {
+
+    protected UsersResource users() {
+        return testRealmResource().users();
+    }
+
+    protected UserResource user(UserRepresentation user) {
+        if (user.getId()==null) {
+            throw new IllegalStateException("User id cannot be null.");
+        }
+        return user(user.getId());
+    }
+
+    protected UserResource user(String id) {
+        return users().get(id);
+    }
+
+    public static UserRepresentation createUserRep(String username) {
+        UserRepresentation user = new UserRepresentation();
+        user.setUsername(username);
+        user.setEmail(username + "@email.test");
+        return user;
+    }
+
+    public UserRepresentation createUser(UserRepresentation user) {
+        return createUser(users(), user);
+    }
+
+    public UserRepresentation createUser(UsersResource users, UserRepresentation user) {
+        Response response = users.create(user);
+        assertEquals(CREATED.getStatusCode(), response.getStatus());
+        user.setId(getCreatedId(response));
+        response.close();
+        return user;
+    }
+
+}

testsuite/integration-arquillian/tests/other/jpa-performance/README.md

@@ -0,0 +1,48 @@
+# Keycloak JPA Performance Tests
+
+## How to run
+
+1. Build the Arquilian Base Testsuite module: `/testsuite/integration-arquillian/base`
+2. Run the test from this module using `mvn test` or `mvn clean test`.
+
+Optional parameters:
+```
+-Dmany.users.count=10000
+-Dmany.users.batch=1000
+```
+
+### With MySQL
+
+Start dockerized MySQL:
+```
+docker run --name mysql-keycloak -e MYSQL_ROOT_PASSWORD=keycloak -e MYSQL_DATABASE=keycloak -e MYSQL_USER=keycloak -e MYSQL_PASSWORD=keycloak -d -p 3306:3306 mysql
+```
+
+Additional test parameters:
+```
+-Pclean-jpa
+-Dkeycloak.connectionsJpa.url=jdbc:mysql://localhost/keycloak
+-Dkeycloak.connectionsJpa.driver=com.mysql.jdbc.Driver
+-Dkeycloak.connectionsJpa.user=keycloak
+-Dkeycloak.connectionsJpa.password=keycloak
+```
+
+### With PostgreSQL
+
+Start dockerized PostgreSQL:
+```
+docker run --name postgres-keycloak -e POSTGRES_PASSWORD=keycloak -d -p 5432:5432 postgres
+```
+
+Additional test parameters:
+```
+-Pclean-jpa
+-Dkeycloak.connectionsJpa.url=jdbc:postgresql://localhost/postgres
+-Dkeycloak.connectionsJpa.driver=org.postgresql.Driver
+-Dkeycloak.connectionsJpa.user=postgres
+-Dkeycloak.connectionsJpa.password=keycloak
+```
+
+## Reports
+
+Test creates reports in `target/stats`.

testsuite/integration-arquillian/tests/other/jpa-performance/pom.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-tests-other</artifactId>
+        <version>1.9.1.Final-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>integration-arquillian-tests-jpa-performance</artifactId>
+
+    <name>Keycloak JPA Performance Tests</name>
+    
+</project>

testsuite/integration-arquillian/tests/other/jpa-performance/src/test/java/org/keycloak/testsuite/user/ManyUsersTest.java

@@ -0,0 +1,119 @@
+package org.keycloak.testsuite.user;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.util.Timer;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.util.JsonSerialization;
+import static org.junit.Assert.fail;
+import org.keycloak.admin.client.resource.RealmResource;
+import static org.keycloak.testsuite.util.IOUtil.PROJECT_BUILD_DIRECTORY;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class ManyUsersTest extends AbstractUserTest {
+
+    private static final int COUNT = Integer.parseInt(System.getProperty("many.users.count", "10000"));
+    private static final int BATCH = Integer.parseInt(System.getProperty("many.users.batch", "1000"));
+
+    private static final String REALM = "realm_with_many_users";
+
+    private List<UserRepresentation> users;
+
+    private final Timer realmTimer = new Timer();
+    private final Timer usersTimer = new Timer();
+
+    protected RealmResource realmResource() {
+        return realmsResouce().realm(REALM);
+    }
+
+    @Before
+    public void before() {
+        users = new LinkedList<>();
+        for (int i = 0; i < COUNT; i++) {
+            users.add(createUserRep("user" + i));
+        }
+
+        realmTimer.reset("create realm before test");
+        RealmRepresentation realm = new RealmRepresentation();
+        realm.setRealm(REALM);
+        realmsResouce().create(realm);
+    }
+
+    @After
+    public void after() {
+        realmTimer.clearStats(true, true, false);
+        usersTimer.clearStats();
+    }
+
+    @Test
+    public void manyUsers() throws IOException {
+        RealmRepresentation realm = realmResource().toRepresentation();
+        realm.setUsers(users);
+
+        // CREATE 
+        realmTimer.reset("create " + users.size() + " users");
+        usersTimer.reset("create " + BATCH + " users");
+        int i = 0;
+        for (UserRepresentation user : users) {
+            createUser(realmResource().users(), user);
+            if (++i % BATCH == 0) {
+                usersTimer.reset();
+                log.info("Created users: " + i + " / " + users.size());
+            }
+        }
+        if (i % BATCH != 0) {
+            usersTimer.reset();
+            log.info("Created users: " + i + " / " + users.size());
+        }
+
+        // SAVE REALM
+        realmTimer.reset("save realm with " + users.size() + " users");
+        File realmFile = new File(PROJECT_BUILD_DIRECTORY, REALM + ".json");
+        JsonSerialization.writeValueToStream(new BufferedOutputStream(new FileOutputStream(realmFile)), realm);
+
+        // DELETE REALM
+        realmTimer.reset("delete realm with " + users.size() + " users");
+        realmResource().remove();
+        try {
+            realmResource().toRepresentation();
+            fail("realm not deleted");
+        } catch (Exception ex) {
+            log.debug("realm deleted");
+        }
+
+        // RE-IMPORT SAVED REALM
+        realmTimer.reset("re-import realm with " + realm.getUsers().size() + " users");
+        realmsResouce().create(realm);
+        realmTimer.reset("load " + realm.getUsers().size() + " users");
+        users = realmResource().users().search("", 0, Integer.MAX_VALUE);
+
+        // DELETE INDIVIDUAL USERS
+        realmTimer.reset("delete " + users.size() + " users");
+        usersTimer.reset("delete " + BATCH + " users", false);
+        i = 0;
+        for (UserRepresentation user : users) {
+            realmResource().users().get(user.getId()).remove();
+            if (++i % BATCH == 0) {
+                usersTimer.reset();
+                log.info("Deleted users: " + i + " / " + users.size());
+            }
+        }
+        if (i % BATCH != 0) {
+            usersTimer.reset();
+            log.info("Deleted users: " + i + " / " + users.size());
+        }
+        realmTimer.reset();
+    }
+
+}

testsuite/integration-arquillian/tests/other/pom.xml

@@ -146,6 +146,12 @@
                 <module>mod_auth_mellon</module>
             </modules>
         </profile>
+        <profile>
+            <id>jpa-performance</id>
+            <modules>
+                <module>jpa-performance</module>
+            </modules>
+        </profile>
     </profiles>
 
 </project>

testsuite/integration-arquillian/tests/pom.xml

@@ -108,6 +108,7 @@
                     <artifactId>maven-surefire-plugin</artifactId>
                     <configuration>
                         <systemPropertyVariables>
+                            <project.build.directory>${project.build.directory}</project.build.directory>
                             <browser>${browser}</browser>
                             <firefox_binary>${firefox_binary}</firefox_binary>
                             <shouldDeploy>false</shouldDeploy>
@@ -223,6 +224,12 @@
                     <version>2.1.0.Alpha3</version><!-- TODO upgrade <arquillian-graphene.version> and use ${arquillian-graphene.version} -->
                 </dependency>
                 
+                <dependency>
+                    <groupId>jfree</groupId>
+                    <artifactId>jfreechart</artifactId>
+                    <version>1.0.13</version>
+                </dependency>
+                
                 <!--                <dependency>
                     <groupId>org.arquillian.extension</groupId>
                     <artifactId>arquillian-recorder-reporter-impl</artifactId>
@@ -411,10 +418,83 @@
                         <groupId>org.codehaus.mojo</groupId>
                         <artifactId>xml-maven-plugin</artifactId>
                     </plugin>
+                    <plugin>
+                        <groupId>org.liquibase</groupId>
+                        <artifactId>liquibase-maven-plugin</artifactId>
+                    </plugin>
                 </plugins>
             </build>            
         </profile>
         
+        <!-- MySQL -->
+        <profile>
+            <activation>
+                <property>
+                    <name>keycloak.connectionsJpa.driver</name>
+                    <value>com.mysql.jdbc.Driver</value>
+                </property>
+            </activation>
+            <id>mysql</id>
+            <dependencies>
+                <dependency>
+                    <groupId>mysql</groupId>
+                    <artifactId>mysql-connector-java</artifactId>
+                </dependency>
+            </dependencies>
+        </profile>
+
+        <!-- PostgreSQL -->
+        <profile>
+            <activation>
+                <property>
+                    <name>keycloak.connectionsJpa.driver</name>
+                    <value>org.postgresql.Driver</value>
+                </property>
+            </activation>
+            <id>postgresql</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.postgresql</groupId>
+                    <artifactId>postgresql</artifactId>
+                    <version>${postgresql.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
+        
+        <profile>
+            <id>clean-jpa</id>
+            <build>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <groupId>org.liquibase</groupId>
+                            <artifactId>liquibase-maven-plugin</artifactId>
+                            <configuration>
+                                <changeLogFile>META-INF/jpa-changelog-master.xml</changeLogFile>
+
+                                <url>${keycloak.connectionsJpa.url}</url>
+                                <driver>${keycloak.connectionsJpa.driver}</driver>
+                                <username>${keycloak.connectionsJpa.user}</username>
+                                <password>${keycloak.connectionsJpa.password}</password>
+
+                                <promptOnNonLocalDatabase>false</promptOnNonLocalDatabase>
+                                <databaseClass>${keycloak.connectionsJpa.liquibaseDatabaseClass}</databaseClass>
+                            </configuration>
+                            <executions>
+                                <execution>
+                                    <id>clean-jpa</id>
+                                    <phase>clean</phase>
+                                    <goals>
+                                        <goal>dropAll</goal>
+                                    </goals>
+                                </execution>
+                            </executions>
+                        </plugin>
+                    </plugins>
+                </pluginManagement>
+            </build>
+        </profile>        
+        
         <profile>
             <id>auth-server-wildfly</id>
             <properties>