Added UsersJob for generic job related to users. Support for sync/async jobs. Configurable host/port

testsuite/integration/src/main/java/org/keycloak/testutils/KeycloakServer.java

@@ -201,6 +201,10 @@ public class KeycloakServer {
         return server;
     }
 
+    public KeycloakServerConfig getConfig() {
+        return config;
+    }
+
     public void importRealm(InputStream realm) {
         RealmRepresentation rep = loadJson(realm, RealmRepresentation.class);
         importRealm(rep);

testsuite/performance-web/README.md

@@ -53,10 +53,20 @@ http://localhost:8081/keycloak-tools/perf/perf-realm/get-users-count?prefix=user
 
 For adding 10000 new users into your database (will start from last added user, so you don't need to explicitly check how many users to create are needed:
 ```shell 
-http://localhost:8081/keycloak-tools/perf/perf-realm/create-available-users?prefix=user&count=10000&batch=100&roles=user
+http://localhost:8081/keycloak-tools/perf/perf-realm/create-available-users?prefix=user&count=10000&batch=100&async=true&roles=role-0,role-1
 ````
 
-Seeing progress of job for creating users
+For update role mappings of all users:
+```shell 
+http://localhost:8081/keycloak-tools/perf/perf-realm/update-all-users?prefix=user&async=true&roles=role-3,perf-app:approle-3,perf-app:approle-4
+````
+
+For deleting all users:
+```shell 
+http://localhost:8081/keycloak-tools/perf/perf-realm/delete-all-users?prefix=user
+````
+
+Seeing progress of job for creating/updating/deleting users
 ```shell 
 http://localhost:8081/keycloak-tools/perf/jobs
 ````

testsuite/performance-web/pom.xml

@@ -216,11 +216,11 @@
                                     <preserveDirectories>false</preserveDirectories>
 
                                     <requestGroups>
-                                        <requestGroup>
+                                        <requestGroup implementation="com.lazerycode.jmeter.analyzer.config.RequestGroup">
                                             <name>aggregatedRequests</name>
                                             <pattern>* request</pattern>
                                         </requestGroup>
-                                        <requestGroup>
+                                        <requestGroup implementation="com.lazerycode.jmeter.analyzer.config.RequestGroup">
                                             <name>codes</name>
                                             <pattern>**/perf-app/perf-servlet?code=*</pattern>
                                         </requestGroup>

testsuite/performance-web/src/main/java/org/keycloak/testsuite/performance/web/KeycloakPerfServer.java

@@ -85,6 +85,7 @@ public class KeycloakPerfServer {
 
         ServletInfo servlet = new ServletInfo("PerfAppServlet", PerfAppServlet.class);
         servlet.addMapping("/perf-servlet/*");
+        servlet.addInitParam(PerfAppServlet.BASE_URL_INIT_PARAM, "http://" + keycloakServer.getConfig().getHost() + ":" + keycloakServer.getConfig().getPort());
 
         deploymentInfo.addServlet(servlet);
 

testsuite/performance-web/src/main/java/org/keycloak/testsuite/performance/web/OAuthClient.java

@@ -42,7 +42,7 @@ import org.keycloak.util.BasicAuthHelper;
  */
 public class OAuthClient {
 
-    private String baseUrl = "http://localhost:8081/auth";
+    private String baseUrl;
 
     private String realm = "perf-realm";
 
@@ -52,16 +52,19 @@ public class OAuthClient {
 
     private String clientId = "perf-app";
 
-    private String redirectUri = "http://localhost:8081/perf-app/perf-servlet";
+    private String redirectUri;
 
     private String state = "123";
 
     private PublicKey realmPublicKey;
 
-    public OAuthClient() {
+    public OAuthClient(String baseUrl) {
         try {
             JSONObject realmJson = new JSONObject(IOUtils.toString(getClass().getResourceAsStream("/perfrealm.json")));
             realmPublicKey = PemUtils.decodePublicKey(realmJson.getString("publicKey"));
+
+            this.baseUrl = (baseUrl != null) ? baseUrl + "/auth" : "http://localhost:8081/auth";
+            this.redirectUri = baseUrl + "/perf-app/perf-servlet";
         } catch (Exception e) {
             throw new RuntimeException("Failed to retrieve realm public key", e);
         }

testsuite/performance-web/src/main/java/org/keycloak/testsuite/performance/web/PerfAppServlet.java

@@ -25,6 +25,8 @@ import org.keycloak.util.Time;
  */
 public class PerfAppServlet extends HttpServlet {
 
+    public static final String BASE_URL_INIT_PARAM = "baseUrl";
+
     private Template indexTemplate;
     private OAuthClient oauthClient;
 
@@ -35,7 +37,8 @@ public class PerfAppServlet extends HttpServlet {
             cfg.setTemplateLoader(new ClassTemplateLoader(getClass(), "/"));
             indexTemplate = cfg.getTemplate("perf-app-resources/index.ftl");
 
-            oauthClient = new OAuthClient();
+            String baseUrl = getInitParameter(BASE_URL_INIT_PARAM);
+            oauthClient = new OAuthClient(baseUrl);
         } catch (IOException ioe) {
             throw new ServletException(ioe);
         }

testsuite/performance-web/src/main/resources/perfrealm.json

@@ -8,7 +8,6 @@
     "privateKey": "MIICXAIBAAKBgQCrVrCuTtArbgaZzL1hvh0xtL5mc7o0NqPVnYXkLvgcwiC3BjLGw1tGEGoJaXDuSaRllobm53JBhjx33UNv+5z/UMG4kytBWxheNVKnL6GgqlNabMaFfPLPCF8kAgKnsi79NMo+n6KnSY8YeUmec/p2vjO2NjsSAVcWEQMVhJ31LwIDAQABAoGAfmO8gVhyBxdqlxmIuglbz8bcjQbhXJLR2EoS8ngTXmN1bo2L90M0mUKSdc7qF10LgETBzqL8jYlQIbt+e6TH8fcEpKCjUlyq0Mf/vVbfZSNaVycY13nTzo27iPyWQHK5NLuJzn1xvxxrUeXI6A2WFpGEBLbHjwpx5WQG9A+2scECQQDvdn9NE75HPTVPxBqsEd2z10TKkl9CZxu10Qby3iQQmWLEJ9LNmy3acvKrE3gMiYNWb6xHPKiIqOR1as7L24aTAkEAtyvQOlCvr5kAjVqrEKXalj0Tzewjweuxc0pskvArTI2Oo070h65GpoIKLc9jf+UA69cRtquwP93aZKtW06U8dQJAF2Y44ks/mK5+eyDqik3koCI08qaC8HYq2wVl7G2QkJ6sbAaILtcvD92ToOvyGyeE0flvmDZxMYlvaZnaQ0lcSQJBAKZU6umJi3/xeEbkJqMfeLclD27XGEFoPeNrmdx0q10Azp4NfJAY+Z8KRyQCR2BEG+oNitBOZ+YXF9KCpH3cdmECQHEigJhYg+ykOvr1aiZUMFT72HU0jnmQe2FVekuG+LJUt2Tm7GtMjTFoGpf0JwrVuZN39fOYAlo+nTixgeW7X8Y=",
     "publicKey": "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrVrCuTtArbgaZzL1hvh0xtL5mc7o0NqPVnYXkLvgcwiC3BjLGw1tGEGoJaXDuSaRllobm53JBhjx33UNv+5z/UMG4kytBWxheNVKnL6GgqlNabMaFfPLPCF8kAgKnsi79NMo+n6KnSY8YeUmec/p2vjO2NjsSAVcWEQMVhJ31LwIDAQAB",
     "requiredCredentials": [ "password" ],
-    "defaultRoles": [ "user" ],
     "smtpServer": {
         "from": "auto@keycloak.org",
         "host": "localhost",
@@ -30,7 +29,7 @@
             "name" : "third-party",
             "enabled": true,
             "redirectUris": [
-                "http://localhost:8081/app/*"
+                "/app/*"
             ],
             "secret": "password"
         }
@@ -48,41 +47,55 @@
         },
         {
             "client": "perf-app",
-            "roles": ["user"]
+            "roles": [ "role-0", "role-1", "role-2", "role-3", "role-4" ]
         }
     ],
     "applications": [
         {
             "name": "perf-app",
             "enabled": true,
-            "baseUrl": "http://localhost:8081/perf-app",
+            "baseUrl": "/perf-app",
             "redirectUris": [
-                "http://localhost:8081/perf-app/*"
+                "/perf-app/*"
             ],
-            "adminUrl": "http://localhost:8081/perf-app/perf-servlet",
+            "adminUrl": "/perf-app/perf-servlet",
             "secret": "password"
          }
     ],
     "roles" : {
         "realm" : [
             {
-                "name": "user",
+                "name": "role-0"
-                "description": "Have User privileges"
             },
             {
-                "name": "admin",
+                "name": "role-1"
-                "description": "Have Administrator privileges"
+            },
+            {
+                "name": "role-2"
+            },
+            {
+                "name": "role-3"
+            },
+            {
+                "name": "role-4"
             }
         ],
         "application" : {
             "perf-app" : [
                 {
-                    "name": "customer-user",
+                    "name": "approle-0"
-                    "description": "Have Customer User privileges"
                 },
                 {
-                    "name": "customer-admin",
+                    "name": "approle-1"
-                    "description": "Have Customer Admin privileges"
+                },
+                {
+                    "name": "approle-2"
+                },
+                {
+                    "name": "approle-3"
+                },
+                {
+                    "name": "approle-4"
                 }
             ]
         }

testsuite/performance-web/src/test/jmeter/keycloak_web_perf_test.jmx

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<jmeterTestPlan version="1.2" properties="2.6" jmeter="2.11 r1554548">
+<jmeterTestPlan version="1.2" properties="2.5" jmeter="2.10 r1533061">
   <hashTree>
     <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true">
       <stringProp name="TestPlan.comments"></stringProp>
@@ -166,7 +166,7 @@
           <stringProp name="HTTPSampler.response_timeout"></stringProp>
           <stringProp name="HTTPSampler.protocol"></stringProp>
           <stringProp name="HTTPSampler.contentEncoding"></stringProp>
-          <stringProp name="HTTPSampler.path">/auth/realms/perf-realm/tokens/auth/request/login?response_type=code&amp;redirect_uri=http%3A%2F%2Flocalhost%3A8081%2Fperf-app%2Fperf-servlet&amp;state=123&amp;client_id=perf-app</stringProp>
+          <stringProp name="HTTPSampler.path">/auth/realms/perf-realm/tokens/auth/request/login?response_type=code&amp;redirect_uri=http%3A%2F%2F${host}%3A${port}%2Fperf-app%2Fperf-servlet&amp;state=123&amp;client_id=perf-app</stringProp>
           <stringProp name="HTTPSampler.method">POST</stringProp>
           <boolProp name="HTTPSampler.follow_redirects">true</boolProp>
           <boolProp name="HTTPSampler.auto_redirects">false</boolProp>
@@ -322,6 +322,32 @@
             <bytes>true</bytes>
           </value>
         </objProp>
+        <objProp>
+          <value class="SampleSaveConfiguration">
+            <time>true</time>
+            <latency>true</latency>
+            <timestamp>true</timestamp>
+            <success>true</success>
+            <label>true</label>
+            <code>true</code>
+            <message>true</message>
+            <threadName>true</threadName>
+            <dataType>true</dataType>
+            <encoding>false</encoding>
+            <assertions>true</assertions>
+            <subresults>true</subresults>
+            <responseData>false</responseData>
+            <samplerData>false</samplerData>
+            <xml>false</xml>
+            <fieldNames>false</fieldNames>
+            <responseHeaders>false</responseHeaders>
+            <requestHeaders>false</requestHeaders>
+            <responseDataOnError>false</responseDataOnError>
+            <saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
+            <assertionsResultsToSave>0</assertionsResultsToSave>
+            <bytes>true</bytes>
+          </value>
+        </objProp>
         <stringProp name="filename"></stringProp>
       </ResultCollector>
       <hashTree/>

testsuite/tools/src/main/java/org/keycloak/test/tools/PerfTools.java

@@ -5,8 +5,11 @@ import org.keycloak.exportimport.ExportImportProvider;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.RealmModel;
-import org.keycloak.models.UserModel;
+import org.keycloak.test.tools.jobs.CreateUsersJob;
-import org.keycloak.test.tools.jobs.CreateUsers;
+import org.keycloak.test.tools.jobs.DeleteUsersJob;
+import org.keycloak.test.tools.jobs.UpdateUsersJob;
+import org.keycloak.test.tools.jobs.UsersJob;
+import org.keycloak.test.tools.jobs.UsersJobInitializer;
 import org.keycloak.util.ProviderLoader;
 
 import javax.ws.rs.GET;
@@ -18,11 +21,11 @@ import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 
-import java.util.HashMap;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -41,7 +44,7 @@ public class PerfTools {
     @Context
     private KeycloakSession session;
 
-    private List<Job> jobs = new LinkedList<Job>();
+    private List<JobRepresentation> jobs = new LinkedList<JobRepresentation>();
 
     public PerfTools(KeycloakSessionFactory sessionFactory) {
         this.sessionFactory = sessionFactory;
@@ -50,16 +53,16 @@ public class PerfTools {
     @GET
     @Path("jobs")
     @Produces("application/json")
-    public List<Job> jobs() {
+    public List<JobRepresentation> jobs() {
         return jobs;
     }
 
     @GET
     @Path("delete-jobs")
     public void deleteJobs() {
-        Iterator<Job> itr = jobs.iterator();
+        Iterator<JobRepresentation> itr = jobs.iterator();
         while(itr.hasNext()) {
-            Job j = itr.next();
+            JobRepresentation j = itr.next();
             if (j.getError() != null || j.getCount() == j.getTotal()) {
                 itr.remove();
             }
@@ -68,7 +71,116 @@ public class PerfTools {
 
     @GET
     @Path("{realm}/create-users")
-    public void createUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count, @QueryParam("batch") Integer batch, @QueryParam("start") Integer start, @QueryParam("prefix") String prefix, @QueryParam("roles") String roles) throws InterruptedException {
+    public void createUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count,
+                            @QueryParam("batch") Integer batch, @QueryParam("start") Integer start, @QueryParam("prefix") String prefix,
+                            @QueryParam("async") Boolean async, @QueryParam("roles") String roles) throws InterruptedException {
+        final String[] rolesArray = roles != null ? roles.split(",") : new String[0];
+
+        createAndRunJob(realmName, count, batch, start, prefix, async, "Create users", new UsersJobInitializer() {
+
+            @Override
+            public UsersJob instantiateJob() {
+                return new CreateUsersJob(rolesArray);
+            }
+
+        });
+    }
+
+    // Same as createUsers, but dynamically compute "start" (Next available user)
+    @GET
+    @Path("{realm}/create-available-users")
+    public void createAvailableUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count,
+                                     @QueryParam("batch") Integer batch, @QueryParam("prefix") String prefix,
+                                     @QueryParam("async") Boolean async, @QueryParam("roles") String roles) throws InterruptedException {
+        int start = getUsersCount(realmName, prefix);
+        createUsers(realmName, count, batch, start, prefix, async, roles);
+    }
+
+    @GET
+    @Path("{realm}/delete-users")
+    public void deleteUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count,
+                            @QueryParam("batch") Integer batch, @QueryParam("start") Integer start, @QueryParam("prefix") String prefix,
+                            @QueryParam("async") Boolean async) throws InterruptedException {
+        createAndRunJob(realmName, count, batch, start, prefix, async, "Delete users", new UsersJobInitializer() {
+
+            @Override
+            public UsersJob instantiateJob() {
+                return new DeleteUsersJob();
+            }
+
+        });
+    }
+
+    @GET
+    @Path("{realm}/delete-all-users")
+    public void deleteUsers(@PathParam("realm") String realmName, @QueryParam("prefix") String prefix, @QueryParam("async") Boolean async) throws InterruptedException {
+        int count = getUsersCount(realmName, prefix);
+        if (count == 0) {
+            return;
+        }
+
+        int batch = count / 10;
+        if (batch == 0) {
+            batch = 1;
+        }
+
+        deleteUsers(realmName, count, batch, 0, prefix, async);
+    }
+
+    @GET
+    @Path("{realm}/update-users")
+    public void updateUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count,
+                            @QueryParam("batch") Integer batch, @QueryParam("start") Integer start, @QueryParam("prefix") String prefix,
+                            @QueryParam("async") Boolean async, @QueryParam("roles") String roles) throws InterruptedException {
+        final String[] rolesArray = roles != null ? roles.split(",") : new String[0];
+
+        createAndRunJob(realmName, count, batch, start, prefix, async, "Update users", new UsersJobInitializer() {
+
+            @Override
+            public UsersJob instantiateJob() {
+                return new UpdateUsersJob(rolesArray);
+            }
+
+        });
+    }
+
+    @GET
+    @Path("{realm}/update-all-users")
+    public void updateAllUsers(@PathParam("realm") String realmName, @QueryParam("prefix") String prefix, @QueryParam("async") Boolean async,
+                               @QueryParam("roles") String roles) throws InterruptedException {
+        int count = getUsersCount(realmName, prefix);
+        if (count == 0) {
+            return;
+        }
+
+        int batch = count / 10;
+        if (batch == 0) {
+            batch = 1;
+        }
+
+        updateUsers(realmName, count, batch, 0, prefix, async, roles);
+    }
+
+
+    @GET
+    @Path("{realm}/get-users-count")
+    public Response getUsersCountReq(@PathParam("realm") String realmName, @QueryParam("prefix") String prefix) {
+        int usersCount = getUsersCount(realmName, prefix);
+        return Response.ok(String.valueOf(usersCount)).build();
+    }
+
+    private int getUsersCount(String realmName, String prefix) {
+        RealmModel realm = session.getRealmByName(realmName);
+
+        // TODO: method for count on model
+        if (prefix == null) {
+            return realm.getUsers().size();
+        } else {
+            return realm.searchForUser(prefix).size();
+        }
+    }
+
+    private void createAndRunJob(String realmName, Integer count, Integer batch, Integer start, String prefix, Boolean async, String jobName, UsersJobInitializer initializer) throws InterruptedException {
         if (count == null) {
             count = 1;
         }
@@ -81,51 +193,34 @@ public class PerfTools {
         if (prefix == null) {
             prefix = String.valueOf(System.currentTimeMillis());
         }
+        if (async == null) {
+            async = true;
+        }
 
-        String[] rolesArray = roles != null ? roles.split(",") : new String[0];
+        int executorsCount = count / batch;
+        if (count % batch > 0) {
+            executorsCount++;
+        }
+        CountDownLatch latch = new CountDownLatch(executorsCount);
 
-        Job job = new Job("Create users " + prefix + "-" + start + " to " + prefix + "-" + (start + count), count);
+        JobRepresentation job = new JobRepresentation(jobName + " " + prefix + "-" + start + " to " + prefix + "-" + (start + count), count);
         jobs.add(job);
 
+        List<UsersJob> usersJobs = new ArrayList<UsersJob>();
         for (int s = start; s < (start + count); s += batch) {
             int c = s + batch <= (start + count) ? batch : (start + count) - s;
-            executor.submit(new CreateUsers(job, sessionFactory, realmName, s, c, prefix, rolesArray));
+            UsersJob usersJob = initializer.instantiateJob();
-        }
+            usersJob.init(job, sessionFactory, realmName, s, c, prefix, latch);
+            usersJobs.add(usersJob);
         }
 
-    @GET
+        // Run executors once all are initialized
-    @Path("{realm}/delete-users")
+        for (UsersJob usersJob : usersJobs) {
-    public void deleteUsers(@PathParam("realm") String realmName) {
+            executor.submit(usersJob);
-        RealmModel realm = session.getRealmByName(realmName);
-        for (UserModel user : realm.getUsers()) {
-            realm.removeUser(user.getUsername());
-        }
         }
 
-
+        if (!async) {
-    @GET
+            latch.await();
-    @Path("{realm}/get-users-count")
-    public Response getUsersCountReq(@PathParam("realm") String realmName, @QueryParam("prefix") String prefix) {
-        int usersCount = getUsersCount(realmName, prefix);
-        return Response.ok(String.valueOf(usersCount)).build();
-    }
-
-    // Same as createUsers, but dynamically compute "start" (Next available user)
-    @GET
-    @Path("{realm}/create-available-users")
-    public void createAvailableUsers(@PathParam("realm") String realmName, @QueryParam("count") Integer count, @QueryParam("batch") Integer batch, @QueryParam("prefix") String prefix, @QueryParam("roles") String roles) throws InterruptedException {
-        int start = getUsersCount(realmName, prefix);
-        createUsers(realmName, count, batch, start, prefix, roles);
-    }
-
-    private int getUsersCount(String realmName, String prefix) {
-        RealmModel realm = session.getRealmByName(realmName);
-
-        // TODO: method for count on model
-        if (prefix == null) {
-            return realm.getUsers().size();
-        } else {
-            return realm.searchForUser(prefix).size();
         }
     }
 
@@ -146,7 +241,7 @@ public class PerfTools {
         }
     }
 
-    public class Job {
+    public static class JobRepresentation {
         private final String description;
         private final int total;
         private AtomicInteger count = new AtomicInteger();
@@ -154,7 +249,7 @@ public class PerfTools {
         private AtomicLong started = new AtomicLong();
         private AtomicLong completed = new AtomicLong();
 
-        public Job(String description, int total) {
+        public JobRepresentation(String description, int total) {
             this.description = description;
             this.total = total;
         }

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/CreateUsers.java

@@ -1,77 +0,0 @@
-package org.keycloak.test.tools.jobs;
-
-import org.keycloak.models.KeycloakSession;
-import org.keycloak.models.KeycloakSessionFactory;
-import org.keycloak.models.RealmModel;
-import org.keycloak.models.UserCredentialModel;
-import org.keycloak.models.UserModel;
-import org.keycloak.services.managers.RealmManager;
-import org.keycloak.test.tools.PerfTools;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-/**
- * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
- */
-public class CreateUsers implements Runnable {
-
-    private PerfTools.Job job;
-    private final KeycloakSessionFactory sessionFactory;
-    private final String realmName;
-    private int start;
-    private int count;
-    private String prefix;
-    private String[] roles;
-
-    public CreateUsers(PerfTools.Job job, KeycloakSessionFactory sessionFactory, String realmName, int start, int count, String prefix, String[] roles) {
-        this.job = job;
-        this.sessionFactory = sessionFactory;
-        this.realmName = realmName;
-        this.start = start;
-        this.count = count;
-        this.prefix = prefix;
-        this.roles = roles;
-    }
-
-    @Override
-    public void run() {
-        job.start();
-
-        KeycloakSession session = sessionFactory.create();
-        try {
-            session.getTransaction().begin();
-
-            RealmModel realm = new RealmManager(session).getRealmByName(realmName);
-
-            for (int i = start; i < (start + count); i++) {
-                UserModel user = realm.addUser(prefix + "-" + i);
-                user.setEnabled(true);
-                user.setFirstName("First");
-                user.setLastName("Last");
-                user.setEmail(prefix + "-" + i + "@localhost");
-
-                UserCredentialModel password = new UserCredentialModel();
-                password.setType(UserCredentialModel.PASSWORD);
-                password.setValue("password");
-
-                user.updateCredential(password);
-
-                for (String r : roles) {
-                    user.grantRole(realm.getRole(r));
-                }
-
-                job.increment();
-            }
-
-            session.getTransaction().commit();
-        } catch (Throwable t) {
-            StringWriter sw = new StringWriter();
-            t.printStackTrace(new PrintWriter(sw));
-            job.setError(sw.toString());
-        } finally {
-            session.close();
-        }
-    }
-
-}

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/CreateUsersJob.java

@@ -0,0 +1,50 @@
+package org.keycloak.test.tools.jobs;
+
+import org.keycloak.models.ApplicationModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserCredentialModel;
+import org.keycloak.models.UserModel;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
+ */
+public class CreateUsersJob extends UsersJob {
+
+    private String[] roles;
+
+    public CreateUsersJob(String[] roles) {
+        this.roles = roles;
+    }
+
+    @Override
+    protected void before(KeycloakSession session) {
+    }
+
+    @Override
+    protected void runIteration(RealmModel realm, Map<String, ApplicationModel> apps, Set<RoleModel> realmRoles, Map<String, Set<RoleModel>> appRoles, int counter) {
+        String username = prefix + "-" + counter;
+        UserModel user = realm.addUser(username);
+        user.setEnabled(true);
+        user.setFirstName("First");
+        user.setLastName("Last");
+        user.setEmail(username + "@localhost");
+
+        UserCredentialModel password = new UserCredentialModel();
+        password.setType(UserCredentialModel.PASSWORD);
+        password.setValue("password");
+
+        user.updateCredential(password);
+
+        for (String r : roles) {
+            grantRole(user, r, realmRoles, appRoles);
+        }
+    }
+
+}

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/DeleteUsersJob.java

@@ -0,0 +1,38 @@
+package org.keycloak.test.tools.jobs;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.keycloak.models.ApplicationModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserModel;
+import org.keycloak.services.managers.RealmManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class DeleteUsersJob extends UsersJob {
+
+    private Iterator<UserModel> users;
+
+    @Override
+    protected void before(KeycloakSession session) {
+        RealmModel realm = new RealmManager(session).getRealmByName(realmName);
+
+        // TODO: pagination
+        List<UserModel> users = (prefix==null) ? realm.getUsers() : realm.searchForUser(prefix);
+        users = users.subList(start, start + count);
+
+        this.users = users.iterator();
+    }
+
+    @Override
+    protected void runIteration(RealmModel realm, Map<String, ApplicationModel> apps, Set<RoleModel> realmRoles, Map<String, Set<RoleModel>> appRoles, int counter) {
+        String username = users.next().getUsername();
+        realm.removeUser(username);
+    }
+}

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/UpdateUsersJob.java

@@ -0,0 +1,54 @@
+package org.keycloak.test.tools.jobs;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.keycloak.models.ApplicationModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserModel;
+import org.keycloak.services.managers.RealmManager;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class UpdateUsersJob extends UsersJob {
+
+    private String[] roles;
+    private Iterator<UserModel> users;
+
+    public UpdateUsersJob(String[] roles) {
+        this.roles = roles;
+    }
+
+    @Override
+    protected void before(KeycloakSession session) {
+        RealmModel realm = new RealmManager(session).getRealmByName(realmName);
+
+        // TODO: pagination
+        List<UserModel> users = (prefix==null) ? realm.getUsers() : realm.searchForUser(prefix);
+        users = users.subList(start, start + count);
+
+        this.users = users.iterator();
+    }
+
+    @Override
+    protected void runIteration(RealmModel realm, Map<String, ApplicationModel> apps, Set<RoleModel> realmRoles, Map<String, Set<RoleModel>> appRoles, int counter) {
+        String username = users.next().getUsername();
+
+        // Remove all role mappings first
+        UserModel user = realm.getUser(username);
+        Set<RoleModel> currRoles = user.getRoleMappings();
+        for (RoleModel role : currRoles) {
+            user.deleteRoleMapping(role);
+        }
+
+        // Add new roles now
+        for (String r : roles) {
+            grantRole(user, r, realmRoles, appRoles);
+        }
+    }
+}

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/UsersJob.java

@@ -0,0 +1,126 @@
+package org.keycloak.test.tools.jobs;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+
+import org.keycloak.models.ApplicationModel;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.RealmModel;
+import org.keycloak.models.RoleModel;
+import org.keycloak.models.UserModel;
+import org.keycloak.services.managers.RealmManager;
+import org.keycloak.test.tools.PerfTools;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public abstract class UsersJob implements Runnable {
+
+    protected PerfTools.JobRepresentation job;
+    protected KeycloakSessionFactory sessionFactory;
+    protected String realmName;
+    protected int start;
+    protected int count;
+    protected String prefix;
+    protected CountDownLatch latch;
+
+    public void init(PerfTools.JobRepresentation job, KeycloakSessionFactory sessionFactory, String realmName, int start, int count, String prefix, CountDownLatch latch) {
+        this.sessionFactory = sessionFactory;
+        this.realmName = realmName;
+        this.start = start;
+        this.count = count;
+        this.prefix = prefix;
+        this.job = job;
+        this.latch = latch;
+
+        KeycloakSession session = sessionFactory.create();
+        try {
+            session.getTransaction().begin();
+
+            before(session);
+
+            session.getTransaction().commit();
+        } catch (Throwable t) {
+            handleThrowable(t, session);
+        } finally {
+            session.close();
+        }
+    }
+
+    @Override
+    public void run() {
+        job.start();
+
+        KeycloakSession session = sessionFactory.create();
+        try {
+            session.getTransaction().begin();
+
+            RealmModel realm = new RealmManager(session).getRealmByName(realmName);
+            Map<String, ApplicationModel> apps = realm.getApplicationNameMap();
+
+            Set<RoleModel> realmRoles = realm.getRoles();
+            Map<String, Set<RoleModel>> appRoles = new HashMap<String, Set<RoleModel>>();
+            for (Map.Entry<String, ApplicationModel> appEntry : apps.entrySet()) {
+                appRoles.put(appEntry.getKey(), appEntry.getValue().getRoles());
+            }
+
+            for (int i = start; i < (start + count); i++) {
+                runIteration(realm, apps, realmRoles, appRoles, i);
+                job.increment();
+            }
+
+            session.getTransaction().commit();
+        } catch (Throwable t) {
+            handleThrowable(t, session);
+        } finally {
+            latch.countDown();
+            session.close();
+        }
+
+    }
+
+    protected abstract void before(KeycloakSession keycloakSession);
+
+    protected abstract void runIteration(RealmModel realm, Map<String, ApplicationModel> apps, Set<RoleModel> realmRoles, Map<String, Set<RoleModel>> appRoles, int counter);
+
+    protected RoleModel findRole(Set<RoleModel> roles, String roleName) {
+        for (RoleModel role : roles) {
+            if (role.getName().equals(roleName)) {
+                return role;
+            }
+        }
+
+        return null;
+    }
+
+    protected void grantRole(UserModel user, String roleName, Set<RoleModel> realmRoles, Map<String, Set<RoleModel>> appRoles) {
+        if (roleName.indexOf(':') == -1) {
+            // We expect "realmRoleName"
+            RoleModel realmRole = findRole(realmRoles, roleName);
+            user.grantRole(realmRole);
+        } else {
+            // We expect "appName:appRoleName"
+            String[] parts = roleName.split(":");
+            Set<RoleModel> currentAppRoles = appRoles.get(parts[0]);
+            if (currentAppRoles == null) {
+                throw new IllegalStateException("Application '" + parts[0] + "' not found");
+            }
+
+            RoleModel appRole = findRole(currentAppRoles, parts[1]);
+            user.grantRole(appRole);
+        }
+    }
+
+    private void handleThrowable(Throwable t, KeycloakSession session) {
+        StringWriter sw = new StringWriter();
+        t.printStackTrace(new PrintWriter(sw));
+        job.setError(sw.toString());
+        session.getTransaction().rollback();
+    }
+
+}

testsuite/tools/src/main/java/org/keycloak/test/tools/jobs/UsersJobInitializer.java

@@ -0,0 +1,9 @@
+package org.keycloak.test.tools.jobs;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public interface UsersJobInitializer {
+
+    UsersJob instantiateJob();
+}