Refactoring current performance test

pom.xml

@@ -360,7 +360,7 @@
             <dependency>
                 <groupId>org.apache.jmeter</groupId>
                 <artifactId>ApacheJMeter_java</artifactId>
-                <version>2.9</version>
+                <version>2.10</version>
             </dependency>
             <dependency>
                 <groupId>dom4j</groupId>
@@ -513,7 +513,7 @@
                 <plugin>
                     <groupId>com.lazerycode.jmeter</groupId>
                     <artifactId>jmeter-maven-plugin</artifactId>
-                    <version>1.8.1</version>
+                    <version>1.9.0</version>
                 </plugin>
                 <plugin>
                     <groupId>com.lazerycode.jmeter</groupId>

testsuite/integration/pom.xml

@@ -2,10 +2,10 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <parent>
-        <artifactId>keycloak-parent</artifactId>
+        <artifactId>keycloak-testsuite-pom</artifactId>
         <groupId>org.keycloak</groupId>
         <version>1.0-beta-4-SNAPSHOT</version>
-        <relativePath>../../pom.xml</relativePath>
+        <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

testsuite/performance/README.md

@@ -1,19 +1,16 @@
 Configuration of performance test
 =================================
-- At this moment it's src/test/jmeter/keycloak_perf_test.jmx where you can configure among other things:
--- "ThreadGroup.num_threads" -- number of worker threads
--- "LoopController.loops" -- Number of loops per each thread.
-
 
 - src/test/jmeter/system.properties -- System properties including configuration of providers. Allow to specify:
+-- Number of worker threads and loops to be used by JMeter performance test
 -- which model to use
 -- which test to run
 -- configuration of individual tests. Properties for each test documented in the file
 
 Running performance tests
 =========================
-cd KEYCLOAK_HOME/testsuite
+cd KEYCLOAK_HOME/testsuite/performance
-mvn clean install -DskipTests=true -Pperformance-tests
+mvn clean verify -DskipTests=true -Pperformance-tests
 
 Results:
 - Log is in: testsuite/performance/target/jmeter/logs/keycloak_perf_test.jmx.log
@@ -22,18 +19,26 @@ Results:
 
 Example for running test
 ========================
-Run:
+1) Run:
-mvn clean install -DskipTests=true -Pperformance-tests
+mvn clean verify -DskipTests=true -Pperformance-tests
 with OOTB configuration (Assumption is mongo running on 27017 as it's using mongo by default). This will create 10 new realms.
 
-Then change keycloak_perf_test.jmx to have
+2) Then change src/test/jmeter/system.properties to have
- "ThreadGroup.num_threads" to 10 and
+ "Tkeycloak.jmeter.numThreads" to 10 and
- "LoopController.loops" to 100
+ "keycloak.jmeter.loops" to 100
-And change "keycloak.perf.workerClass" to "org.keycloak.testsuite.performance.CreateUsersWorker" in system.properties
+ "keycloak.perf.workerClass" to "org.keycloak.testsuite.performance.CreateUsersWorker"
 
 Then run again:
-mvn clean install -DskipTests=true -Pperformance-tests
+mvn clean verify -DskipTests=true -Pperformance-tests
-This will create 1000 new users (10 worker threads and each worker doing 100 iterations. Each worker is creating users in separate realm)
+This will create 1000 new users (10 worker threads and each worker doing 100 iterations. Each worker is creating users in separate realm. So 100 users like "user1", "user2", ... "user100" in each realm)
+
+3) Then change src/test/jmeter/system.properties to have
+ "keycloak.perf.workerClass" to "org.keycloak.testsuite.performance.ReadUsersWorker"
+
+Then run again:
+mvn clean verify -DskipTests=true -Pperformance-tests
+ This will read all 1000 previously created users and each user is read 5 times. There are 1000 iterations in total and each iteration is doing 5 read users.
+
 
 TODO: Easier configuration without need to edit config files, more user friendly, easier to configure and run test
 

testsuite/performance/pom.xml

@@ -3,10 +3,10 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>keycloak-parent</artifactId>
+        <artifactId>keycloak-testsuite-pom</artifactId>
         <groupId>org.keycloak</groupId>
         <version>1.0-beta-4-SNAPSHOT</version>
-        <relativePath>../../pom.xml</relativePath>
+        <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -73,9 +73,31 @@
             <groupId>net.iharder</groupId>
             <artifactId>base64</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk16</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.jmeter</groupId>
             <artifactId>ApacheJMeter_java</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcmail-jdk15on</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcmail-jdk15</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.tika</groupId>
+                    <artifactId>tika-parsers</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
 
@@ -161,6 +183,10 @@
                                         <groupId>org.slf4j</groupId>
                                         <artifactId>slf4j-simple</artifactId>
                                     </exclusion>
+                                    <exclusion>
+                                        <groupId>commons-io</groupId>
+                                        <artifactId>commons-io</artifactId>
+                                    </exclusion>
                                 </exclusions>
                             </dependency>
                             <dependency>
@@ -168,6 +194,16 @@
                                 <artifactId>jboss-logging</artifactId>
                                 <version>${jboss.logging.version}</version>
                             </dependency>
+                            <dependency>
+                                <groupId>commons-io</groupId>
+                                <artifactId>commons-io</artifactId>
+                                <version>2.4</version>
+                            </dependency>
+                            <dependency>
+                                <groupId>org.bouncycastle</groupId>
+                                <artifactId>bcprov-jdk16</artifactId>
+                                <version>${bouncycastle.version}</version>
+                            </dependency>
                             <!--
                             <dependency>
                                 <groupId>org.picketlink</groupId>

testsuite/performance/src/test/java/org/keycloak/testsuite/performance/BaseJMeterPerformanceTest.java

@@ -3,9 +3,12 @@ package org.keycloak.testsuite.performance;
 import org.apache.jmeter.protocol.java.sampler.AbstractJavaSamplerClient;
 import org.apache.jmeter.protocol.java.sampler.JavaSamplerContext;
 import org.apache.jmeter.samplers.SampleResult;
+import org.keycloak.Config;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.KeycloakTransaction;
+import org.keycloak.models.RealmModel;
+import org.keycloak.services.managers.RealmManager;
 import org.keycloak.services.resources.KeycloakApplication;
 
 import java.util.concurrent.Callable;
@@ -22,7 +25,34 @@ public class BaseJMeterPerformanceTest extends AbstractJavaSamplerClient {
 
         @Override
         public KeycloakSessionFactory call() throws Exception {
-            return KeycloakApplication.createSessionFactory();
+            KeycloakSessionFactory factory = KeycloakApplication.createSessionFactory();
+
+            // TODO: Workaround due to bouncycastle classpath issues. Should be fixed properly
+            // new ApplianceBootstrap().bootstrap(factory, "/auth");
+            bootstrapAdminRealm(factory, "/auth");
+
+            return factory;
+        }
+
+        private void bootstrapAdminRealm(KeycloakSessionFactory factory, String contextPath) {
+            KeycloakSession keycloakSession = factory.create();
+            keycloakSession.getTransaction().begin();
+
+            try {
+                String adminRealmName = Config.getAdminRealm();
+                if (keycloakSession.getRealm(adminRealmName) == null) {
+
+                    RealmManager manager = new RealmManager(keycloakSession);
+                    manager.setContextPath(contextPath);
+                    RealmModel realm = manager.createRealm(adminRealmName, adminRealmName);
+                    realm.setName(adminRealmName);
+                    realm.setEnabled(true);
+                }
+
+                keycloakSession.getTransaction().commit();
+            } finally {
+                keycloakSession.close();
+            }
         }
 
     });
@@ -42,6 +72,7 @@ public class BaseJMeterPerformanceTest extends AbstractJavaSamplerClient {
         worker = getWorker();
 
         factory = getFactory();
+        getLogger().info("Retrieved factory: " + factory);
         KeycloakSession session = factory.create();
         KeycloakTransaction transaction = session.getTransaction();
         transaction.begin();

testsuite/performance/src/test/java/org/keycloak/testsuite/performance/ReadUsersWorker.java

@@ -96,8 +96,8 @@ public class ReadUsersWorker implements Worker {
 
             // Read scopes of user in realm
             if (readScopes) {
-                ClientModel client = realm.findClient(username);
+//                ClientModel client = realm.findClient(username);
-                client.getScopeMappings();
+//                client.getScopeMappings();
             }
 
             // Validate password (shoould be same as username)

testsuite/performance/src/test/jmeter/keycloak_perf_test.jmx

@@ -15,10 +15,10 @@
         <stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
         <elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true">
           <boolProp name="LoopController.continue_forever">false</boolProp>
-          <stringProp name="LoopController.loops">10</stringProp>
+          <stringProp name="LoopController.loops">${__P(keycloak.jmeter.loops,10)}</stringProp>
         </elementProp>
-        <stringProp name="ThreadGroup.num_threads">1</stringProp>
+        <stringProp name="ThreadGroup.num_threads">${__P(keycloak.jmeter.numThreads,1)}</stringProp>
-        <stringProp name="ThreadGroup.ramp_time">0</stringProp>
+        <stringProp name="ThreadGroup.ramp_time">${__P(keycloak.jmeter.rampTime,0)}</stringProp>
         <longProp name="ThreadGroup.start_time">1362689985000</longProp>
         <longProp name="ThreadGroup.end_time">1362689985000</longProp>
         <boolProp name="ThreadGroup.scheduler">false</boolProp>

testsuite/performance/src/test/jmeter/system.properties

@@ -1,3 +1,9 @@
+# Thread variables used by JMeter. NumThreads is number of worker threads. Loops is number of iterations per each worker. Total number of tests is numThreads*loops.
+# reampTime is pause between startup of individual threads
+keycloak.jmeter.numThreads=1
+keycloak.jmeter.loops=10
+keycloak.jmeter.rampTime=0
+
 ## Choose implementation of KeycloakSessionFactory
 # keycloak.model.provider=jpa
 keycloak.model.provider=mongo
@@ -25,7 +31,7 @@ keycloak.perf.workerClass=org.keycloak.testsuite.performance.CreateRealmsWorker
 ## Properties for CreateRealms test. This test is used to create some realms.
 # Each iteration of single worker thread will add one realm and it will add some roles, defaultRoles, credentials and applications to it
 # Offset where to start creating realms. Count (total number of realms to create) is configurable as number of JMeter threads*loopCount
-# For example: if offset==1 and in JMeter properties we have LoopController.loops=10 and num_threads=2 then we will create 20 realms in total and we will create realms "realm1" - "realm10"
+# For example: if offset==1 and in JMeter properties we have keycloak.jmeter.loops=10 and keycloak.jmeter.numThreads=2 then we will create 20 realms in total and we will create realms "realm1" - "realm10"
 # NOTE: Count (total number of realms to create) is configurable as number of JMeter threads*loopCount
 keycloak.perf.createRealms.realms.offset=1
 # Count of apps per each realm (For example if count=5, we will create apps like "realm1app1" - "realm1app5" for realm "realm1"
@@ -65,8 +71,8 @@ keycloak.perf.createUsers.socialLinksPerUserCount=0
 keycloak.perf.readUsers.realms.offset=1
 # Number of read users in each iteration
 keycloak.perf.readUsers.readUsersPerIteration=5
-# Number of users to read in each realm. After reading all 2000 users, reading will start again from user1
+# Number of users to read in each realm. After reading all 100 users, reading will start again from user1
-keycloak.perf.readUsers.countOfUsersPerRealm=2000
+keycloak.perf.readUsers.countOfUsersPerRealm=100
 keycloak.perf.readUsers.readRoles=true
 keycloak.perf.readUsers.readScopes=true
 keycloak.perf.readUsers.readPassword=true