Merge pull request #5219 from tkyjovsk/KEYCLOAK-6759

KEYCLOAK-6759 Create the databases for import during perf tests

testsuite/performance/README.datasets.md

@@ -0,0 +1,46 @@
+# Keycloak Performance Testsuite - Generating datasets
+
+
+## Generating a set of datasets for multiple realms
+
+The first dataset is small and is created quickly. Building of each subsequent dataset continues on top
+of the previous dataset.
+
+Datasets are created with a specific released server version (rather than a snapshot) in order to be
+usable with later releases - newer server version should be able to migrate schema from any previous release.
+
+We use 10 concurrent threads, which is enough to saturate a 
+dual core machine. For quad-core you can try to double the number of workers.
+
+```
+cd testsuite/performance
+
+mvn clean install -Dserver.version=4.0.0.Beta1
+
+mvn verify -Pteardown
+mvn verify -Pprovision
+mvn verify -Pgenerate-data -Ddataset=10r100u1c -DnumOfWorkers=10
+mvn verify -Pexport-dump -Ddataset=10r100u1c
+
+mvn verify -Pgenerate-data -Ddataset=20r100u1c -DstartAtRealmIdx=10 -DnumOfWorkers=10
+mvn verify -Pexport-dump -Ddataset=20r100u1c
+
+mvn verify -Pgenerate-data -Ddataset=50r100u1c -DstartAtRealmIdx=20 -DnumOfWorkers=10
+mvn verify -Pexport-dump -Ddataset=50r100u1c
+
+mvn verify -Pgenerate-data -Ddataset=200r100u1c -DstartAtRealmIdx=50 -DnumOfWorkers=10
+mvn verify -Pexport-dump -Ddataset=200r100u1c
+
+mvn verify -Pgenerate-data -Ddataset=500r100u1c -DstartAtRealmIdx=200 -DnumOfWorkers=10
+mvn verify -Pexport-dump -Ddataset=500r100u1c
+```
+
+If the dataset dump file is not available locally but it's known that the dataset for specific version exists on the server
+it can be retrieved by specifying a proper server version again. For example:
+```
+mvn verify -Pteardown
+mvn clean install
+mvn verify -Pprovision
+mvn verify -Pimport-dump -Ddataset=20r100u1c -Dserver.version=4.0.0.Beta1
+
+```

testsuite/performance/tests/docker-compose.sh

@@ -399,7 +399,7 @@ case "$OPERATION" in
                 fi
                 echo "Importing $DATASET.sql.gz"
                 set -o pipefail
-                if ! zcat $DATASET.sql.gz | docker exec -i $DB_CONTAINER /usr/bin/mysql -u root --password=root keycloak ; then
+                if ! gunzip -c $DATASET.sql.gz | docker exec -i $DB_CONTAINER /usr/bin/mysql -u root --password=root keycloak ; then
                     echo Import failed.
                     exit 1
                 fi

testsuite/performance/tests/parameters/datasets/10r100u1c.properties

@@ -0,0 +1,8 @@
+numOfRealms=10
+usersPerRealm=100
+clientsPerRealm=1
+realmRoles=100
+realmRolesPerUser=50
+clientRolesPerUser=0
+clientRolesPerClient=0
+hashIterations=27500

testsuite/performance/tests/parameters/datasets/200r100u1c.properties

@@ -0,0 +1,8 @@
+numOfRealms=200
+usersPerRealm=100
+clientsPerRealm=1
+realmRoles=100
+realmRolesPerUser=50
+clientRolesPerUser=0
+clientRolesPerClient=0
+hashIterations=27500

testsuite/performance/tests/parameters/datasets/20r100u1c.properties

@@ -0,0 +1,8 @@
+numOfRealms=20
+usersPerRealm=100
+clientsPerRealm=1
+realmRoles=100
+realmRolesPerUser=50
+clientRolesPerUser=0
+clientRolesPerClient=0
+hashIterations=27500

testsuite/performance/tests/parameters/datasets/500r100u1c.properties

@@ -0,0 +1,8 @@
+numOfRealms=500
+usersPerRealm=100
+clientsPerRealm=1
+realmRoles=100
+realmRolesPerUser=50
+clientRolesPerUser=0
+clientRolesPerClient=0
+hashIterations=27500

testsuite/performance/tests/parameters/datasets/50r100u1c.properties

@@ -0,0 +1,8 @@
+numOfRealms=50
+usersPerRealm=100
+clientsPerRealm=1
+realmRoles=100
+realmRolesPerUser=50
+clientRolesPerUser=0
+clientRolesPerClient=0
+hashIterations=27500

testsuite/performance/tests/pom.xml

@@ -185,6 +185,7 @@
                             <goal>read-project-properties</goal>
                         </goals>
                         <configuration>
+                            <quiet>true</quiet>
                             <files>
                                 <file>${provisioning.properties.file}</file>
                                 <file>${dataset.properties.file}</file>
@@ -456,6 +457,12 @@
         
         <profile>
             <id>generate-data</id>
+            <properties>
+                <startAtRealmIdx>0</startAtRealmIdx>
+                <ignoreConflicts>false</ignoreConflicts>
+                <skipRealmRoles>false</skipRealmRoles>
+                <skipClientRoles>false</skipClientRoles>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -504,6 +511,10 @@
                                         <argument>-DauthUser=${keycloak.admin.user}</argument>
                                         <argument>-DauthPassword=${keycloak.admin.password}</argument>
                                         <argument>-DnumOfWorkers=${numOfWorkers}</argument>
+                                        <argument>-DstartAtRealmIdx=${startAtRealmIdx}</argument>
+                                        <argument>-DignoreConflicts=${ignoreConflicts}</argument>
+                                        <argument>-DskipRealmRoles=${skipRealmRoles}</argument>
+                                        <argument>-DskipClientRoles=${skipClientRoles}</argument>
                                         <argument>org.keycloak.performance.RealmsConfigurationLoader</argument>
                                         <argument>benchmark-realms.json</argument>
                                     </arguments>

testsuite/performance/tests/src/main/java/org/keycloak/performance/RealmsConfigurationLoader.java

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.jboss.logging.Logger;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.representations.idm.ClientRepresentation;
 import org.keycloak.representations.idm.CredentialRepresentation;
@@ -12,6 +13,7 @@ import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.representations.idm.RoleRepresentation;
 import org.keycloak.representations.idm.UserRepresentation;
 
+import javax.ws.rs.ClientErrorException;
 import javax.ws.rs.core.Response;
 import java.io.File;
 import java.io.IOException;
@@ -28,24 +30,37 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.LinkedBlockingQueue;
 import static org.keycloak.performance.RealmsConfigurationBuilder.EXPORT_FILENAME;
 
+import static org.keycloak.performance.TestConfig.ignoreConflicts;
 import static org.keycloak.performance.TestConfig.numOfWorkers;
+import static org.keycloak.performance.TestConfig.skipClientRoles;
+import static org.keycloak.performance.TestConfig.skipRealmRoles;
+import static org.keycloak.performance.TestConfig.startAtRealmIdx;
+import static org.keycloak.performance.TestConfig.startAtUserIdx;
 
 /**
  * # build
- * mvn -f testsuite/integration-arquillian/tests/performance/gatling-perf clean install
+ * mvn -f testsuite/performance/tests clean install
  *
  * # generate benchmark-realms.json file with generated test data
- * mvn -f testsuite/integration-arquillian/tests/performance/gatling-perf exec:java -Dexec.mainClass=org.keycloak.performance.RealmsConfigurationBuilder -DnumOfRealms=2 -DusersPerRealm=2 -DclientsPerRealm=2 -DrealmRoles=2 -DrealmRolesPerUser=2 -DclientRolesPerUser=2 -DclientRolesPerClient=2
+ * mvn -f testsuite/performance/tests exec:java -Dexec.mainClass=org.keycloak.performance.RealmsConfigurationBuilder -DnumOfRealms=2 -DusersPerRealm=2 -DclientsPerRealm=2 -DrealmRoles=2 -DrealmRolesPerUser=2 -DclientRolesPerUser=2 -DclientRolesPerClient=2
  *
  * # use benchmark-realms.json to load the data up to Keycloak Server listening on localhost:8080
- * mvn -f testsuite/integration-arquillian/tests/performance/gatling-perf exec:java -Dexec.mainClass=org.keycloak.performance.RealmsConfigurationLoader -DnumOfWorkers=5 -Dexec.args=benchmark-realms.json > perf-output.txt
+ * mvn -f testsuite/performance/tests exec:java -Dexec.mainClass=org.keycloak.performance.RealmsConfigurationLoader -DnumOfWorkers=5 -Dexec.args=benchmark-realms.json > perf-output.txt
  *
  * @author <a href="mailto:mstrukel@redhat.com">Marko Strukelj</a>
  */
 public class RealmsConfigurationLoader {
 
+    static Logger log = Logger.getLogger(RealmsConfigurationLoader.class.getName());
+
     static final int ERROR_CHECK_INTERVAL = 10;
 
+    static int currentRealm = 0;
+    static int currentUser = 0;
+    static int currentClient = 0;
+
+    static boolean started;
+
     // multi-thread mechanics
     static final BlockingQueue<AdminJob> queue = new LinkedBlockingQueue<>(numOfWorkers);
     static final ArrayList<Worker> workers = new ArrayList<>();
@@ -58,24 +73,28 @@ public class RealmsConfigurationLoader {
     static boolean realmCreated;
 
     public static void main(String [] args) throws IOException {
-        System.out.println("Keycloak servers: "+TestConfig.serverUrisList);
+        println("Keycloak servers: "+TestConfig.serverUrisList);
 
         if (args.length == 0) {
             args = new String[] {EXPORT_FILENAME};
         }
 
         if (args.length != 1) {
-            System.out.println("Usage: java " + RealmsConfigurationLoader.class.getName() + " <FILE>");
+            println("Usage: java " + RealmsConfigurationLoader.class.getName() + " <FILE>");
             return;
         }
 
         String file = args[0];
-        System.out.println("Using file: " + new File(args[0]).getAbsolutePath());
+        println("Using file: " + new File(args[0]).getAbsolutePath());
-        System.out.println("Number of workers (numOfWorkers): " + numOfWorkers);
+        println("Number of workers (numOfWorkers): " + numOfWorkers);
+        println("Parameters: ");
+        println("    startAtRealmIdx: " + startAtRealmIdx);
+//        println("    startAtUserIdx: " + startAtUserIdx);
 
         JsonParser p = initParser(file);
 
         initWorkers();
+        initProgress();
 
         try {
 
@@ -88,6 +107,28 @@ public class RealmsConfigurationLoader {
         }
     }
 
+    private static void initProgress() {
+        Thread t = new Thread(() -> {
+
+            for (;;) {
+                try {
+                    Thread.sleep(60000);
+                    println("At realm: " + currentRealm + ", Clients: " + currentClient + ", Users: " + currentUser);
+                } catch (InterruptedException e) {
+                    return;
+                }
+
+            }
+
+        },"Progress Logger");
+        t.setDaemon(true);
+        t.start();
+    }
+
+    private static void println(String s) {
+        System.out.println(s);
+    }
+
     private static void completeWorkers() {
 
         try {
@@ -101,7 +142,7 @@ public class RealmsConfigurationLoader {
                 try {
                     w.join(5000);
                     if (w.isAlive()) {
-                        System.out.println("Worker thread failed to stop: ");
+                        println("Worker thread failed to stop: ");
                         dumpThread(w);
                     }
                 } catch (InterruptedException e) {
@@ -117,6 +158,7 @@ public class RealmsConfigurationLoader {
         while (t != JsonToken.END_OBJECT && t != JsonToken.END_ARRAY) {
             if (t != JsonToken.START_ARRAY) {
                 readRealm(p);
+                currentRealm += 1;
             }
             t = p.nextToken();
         }
@@ -150,7 +192,7 @@ public class RealmsConfigurationLoader {
         for (StackTraceElement e: w.getStackTrace()) {
             b.append(e.toString()).append("\n");
         }
-        System.out.print(b);
+        println(b.toString());
     }
 
     private static void readRealm(JsonParser p) throws IOException {
@@ -158,57 +200,112 @@ public class RealmsConfigurationLoader {
         // as soon as we encounter users, roles, clients we create a CreateRealmJob
         // TODO: if after that point in a realm we encounter realm attribute, we report a warning but continue
 
-        RealmRepresentation r = new RealmRepresentation();
+        boolean skip = false;
-        JsonToken t = p.nextToken();
+        try {
-        while (t != JsonToken.END_OBJECT) {
+            RealmRepresentation r = new RealmRepresentation();
+            JsonToken t = p.nextToken();
+            outer:
+            while (t != JsonToken.END_OBJECT && !skip) {
 
-            //System.out.println(t + ", name: " + p.getCurrentName() + ", text: '" + p.getText() + "', value: " + p.getValueAsString());
+                //System.out.println(t + ", name: " + p.getCurrentName() + ", text: '" + p.getText() + "', value: " + p.getValueAsString());
 
-            switch (p.getCurrentName()) {
+                switch (p.getCurrentName()) {
-                case "realm":
+                    case "realm":
-                    r.setRealm(getStringValue(p));
+                        r.setRealm(getStringValue(p));
-                    break;
+                        skip = !started && realmSkipped(r.getRealm()) ;
-                case "enabled":
+                        if (skip) {
-                    r.setEnabled(getBooleanValue(p));
+                            break outer;
-                    break;
+                        }
-                case "accessTokenLifespan":
+                        break;
-                    r.setAccessCodeLifespan(getIntegerValue(p));
+                    case "enabled":
-                    break;
+                        r.setEnabled(getBooleanValue(p));
-                case "registrationAllowed":
+                        break;
-                    r.setRegistrationAllowed(getBooleanValue(p));
+                    case "accessTokenLifespan":
-                    break;
+                        r.setAccessCodeLifespan(getIntegerValue(p));
-                case "passwordPolicy":
+                        break;
-                    r.setPasswordPolicy(getStringValue(p));
+                    case "registrationAllowed":
-                    break;
+                        r.setRegistrationAllowed(getBooleanValue(p));
-                case "users":
+                        break;
-                    ensureRealm(r);
+                    case "passwordPolicy":
-                    readUsers(r, p);
+                        r.setPasswordPolicy(getStringValue(p));
-                    break;
+                        break;
-                case "roles":
+                    case "sslRequired":
-                    ensureRealm(r);
+                        r.setSslRequired(getStringValue(p));
-                    readRoles(r, p);
+                        break;
-                    break;
+                    case "users":
-                case "clients":
+                        ensureRealm(r);
-                    ensureRealm(r);
+                        if (seekToStart()) {
-                    readClients(r, p);
+                            enqueueFetchRealmRoles(r);
-                    break;
+                            completePending();
-                default: {
+                        }
-                    // if we don't understand the field we ignore it - but report that
+                        readUsers(r, p);
-                    System.out.println("Realm attribute ignored: " + p.getCurrentName());
+                        break;
-                    consumeAttribute(p);
+                    case "roles":
+                        ensureRealm(r);
+                        readRoles(r, p);
+                        break;
+                    case "clients":
+                        ensureRealm(r);
+                        readClients(r, p);
+                        completePending();
+                        if (seekToStart()) {
+                            enqueueFetchMissingClients(r);
+                            completePending();
+                        }
+                        break;
+                    default: {
+                        // if we don't understand the field we ignore it - but report that
+                        log.warn("Realm attribute ignored: " + p.getCurrentName());
+                        consumeAttribute(p);
+                        continue; // skip p.nextToken() at end of loop - consumeAttribute() already did it
+                    }
                 }
+
+                t = p.nextToken();
             }
-            t = p.nextToken();
+
+            if (skip) {
+                log.info("Realm skipped: " + r.getRealm());
+                consumeParent(p);
+            }
+
+        } finally {
+            // we wait for realm to complete
+            completePending();
+
+            // reset realm specific cache
+            realmCreated = false;
+            clientIdMap.clear();
+            realmRoleIdMap.clear();
+            clientRoleIdMap.clear();
         }
+    }
 
-        // we wait for realm to complete
+    private static void consumeParent(JsonParser p) throws IOException {
-        completePending();
+        while (p.currentToken() != JsonToken.END_OBJECT) {
+            consumeAttribute(p);
+        }
+    }
 
-        // reset realm specific cache
+    private static boolean seekToStart() {
-        realmCreated = false;
+        return startAtRealmIdx > 0 || startAtUserIdx > 0;
-        clientIdMap.clear();
+    }
-        realmRoleIdMap.clear();
+
-        clientRoleIdMap.clear();
+    private static boolean seeking() {
+        return currentRealm < startAtRealmIdx || currentUser < startAtUserIdx;
+    }
+
+    private static boolean realmSkipped(String realm) {
+        int pos = realm.lastIndexOf("_");
+        int idx = Integer.parseInt(realm.substring(pos+1));
+        return idx < startAtRealmIdx;
+    }
+
+    private static boolean userSkipped(String username) {
+        int pos = username.indexOf("_");
+        int end = username.indexOf("_", pos+1);
+        int idx = Integer.parseInt(username.substring(pos+1, end));
+        return idx < startAtUserIdx;
     }
 
     private static void ensureRealm(RealmRepresentation r) {
@@ -220,34 +317,18 @@ public class RealmsConfigurationLoader {
 
     private static void createRealm(RealmRepresentation r) {
         try {
+            started = true;
             queue.put(new CreateRealmJob(r));
         } catch (InterruptedException e) {
             throw new RuntimeException("Interrupted", e);
         }
 
-        // now wait for job to appear
+        completePending();
-        PendingResult next = pendingResult.poll();
-        while (next == null) {
-            waitForAwhile();
-            next = pendingResult.poll();
-        }
-
-        // then wait for the job to complete
-        while (!next.isDone()) {
-            waitForAwhile();
-        }
-
-        try {
-            next.get();
-        } catch (InterruptedException e) {
-            throw new RuntimeException("Interrupted", e);
-        } catch (ExecutionException e) {
-            throw new RuntimeException("Execution failed", e.getCause());
-        }
     }
 
     private static void enqueueCreateUser(RealmRepresentation r, UserRepresentation u) {
         try {
+            started = true;
             queue.put(new CreateUserJob(r, u));
         } catch (InterruptedException e) {
             throw new RuntimeException("Interrupted", e);
@@ -256,6 +337,7 @@ public class RealmsConfigurationLoader {
 
     private static void enqueueCreateRealmRole(RealmRepresentation r, RoleRepresentation role) {
         try {
+            started = true;
             queue.put(new CreateRealmRoleJob(r, role));
         } catch (InterruptedException e) {
             throw new RuntimeException("Interrupted", e);
@@ -264,6 +346,7 @@ public class RealmsConfigurationLoader {
 
     private static void enqueueCreateClientRole(RealmRepresentation r, RoleRepresentation role, String client) {
         try {
+            started = true;
             queue.put(new CreateClientRoleJob(r, role, client));
         } catch (InterruptedException e) {
             throw new RuntimeException("Interrupted", e);
@@ -272,12 +355,31 @@ public class RealmsConfigurationLoader {
 
     private static void enqueueCreateClient(RealmRepresentation r, ClientRepresentation client) {
         try {
+            started = true;
             queue.put(new CreateClientJob(r, client));
         } catch (InterruptedException e) {
             throw new RuntimeException("Interrupted", e);
         }
     }
 
+    private static void enqueueFetchMissingClients(RealmRepresentation r) {
+        try {
+            started = true;
+            queue.put(new FetchMissingClientsJob(r));
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Interrupted", e);
+        }
+    }
+
+    private static void enqueueFetchRealmRoles(RealmRepresentation r) {
+        try {
+            started = true;
+            queue.put(new FetchRealmRolesJob(r));
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Interrupted", e);
+        }
+    }
+
     private static void waitForAwhile() {
         waitForAwhile(100, "Interrupted");
     }
@@ -299,18 +401,22 @@ public class RealmsConfigurationLoader {
         if (t != JsonToken.START_ARRAY) {
             throw new RuntimeException("Error reading field 'users'. Expected array of users [" + t + "]");
         }
-        int count = 0;
+
         t = p.nextToken();
         while (t == JsonToken.START_OBJECT) {
             UserRepresentation u = p.readValueAs(UserRepresentation.class);
-            enqueueCreateUser(r, u);
+            if (!started && userSkipped(u.getUsername())) {
+                log.info("User skipped: " + u.getUsername());
+            } else {
+                enqueueCreateUser(r, u);
+            }
             t = p.nextToken();
-            count += 1;
+            currentUser += 1;
 
             // every some users check to see pending errors
             // in order to short-circuit if any errors have occurred
-            if (count % ERROR_CHECK_INTERVAL == 0) {
+            if (currentUser % ERROR_CHECK_INTERVAL == 0) {
-                checkPendingErrors();
+                checkPendingErrors(u.getUsername());
             }
         }
     }
@@ -367,14 +473,16 @@ public class RealmsConfigurationLoader {
             t = p.nextToken();
             while (t != JsonToken.END_ARRAY) {
                 RoleRepresentation u = p.readValueAs(RoleRepresentation.class);
-                enqueueCreateClientRole(r, u, client);
+                if (!seeking() || !skipClientRoles) {
+                    enqueueCreateClientRole(r, u, client);
+                }
                 t = p.nextToken();
                 count += 1;
 
                 // every some roles check to see pending errors
                 // in order to short-circuit if any errors have occurred
                 if (count % ERROR_CHECK_INTERVAL == 0) {
-                    checkPendingErrors();
+                    checkPendingErrors(u.getName());
                 }
             }
             t = p.nextToken();
@@ -393,14 +501,16 @@ public class RealmsConfigurationLoader {
         int count = 0;
         while (t == JsonToken.START_OBJECT) {
             RoleRepresentation u = p.readValueAs(RoleRepresentation.class);
-            enqueueCreateRealmRole(r, u);
+            if (!seeking() || !skipRealmRoles) {
+                enqueueCreateRealmRole(r, u);
+            }
             t = p.nextToken();
             count += 1;
 
             // every some roles check to see pending errors
             // in order to short-circuit if any errors have occurred
             if (count % ERROR_CHECK_INTERVAL == 0) {
-                checkPendingErrors();
+                checkPendingErrors(u.getName());
             }
         }
     }
@@ -410,25 +520,25 @@ public class RealmsConfigurationLoader {
         if (t != JsonToken.START_ARRAY) {
             throw new RuntimeException("Error reading field 'clients'. Expected array of clients [" + t + "]");
         }
-        int count = 0;
+
         t = p.nextToken();
         while (t == JsonToken.START_OBJECT) {
             ClientRepresentation u = p.readValueAs(ClientRepresentation.class);
             enqueueCreateClient(r, u);
             t = p.nextToken();
-            count += 1;
+            currentClient += 1;
 
             // every some users check to see pending errors
-            if (count % ERROR_CHECK_INTERVAL == 0) {
+            if (currentClient % ERROR_CHECK_INTERVAL == 0) {
-                checkPendingErrors();
+                checkPendingErrors(u.getClientId());
             }
         }
     }
 
-    private static void checkPendingErrors() {
+    private static void checkPendingErrors(String label) {
         // now wait for job to appear
         PendingResult next = pendingResult.peek();
-        while (next == null) {
+        while (next == null && queue.size() > 0) {
             waitForAwhile();
             next = pendingResult.peek();
         }
@@ -445,7 +555,7 @@ public class RealmsConfigurationLoader {
                 } catch (InterruptedException e) {
                     throw new RuntimeException("Interrupted");
                 } catch (ExecutionException e) {
-                    throw new RuntimeException("Execution failed", e.getCause());
+                    throw new RuntimeException("Execution failed in the vicinity of " + label + ": ", e.getCause());
                 }
             }
         }
@@ -479,9 +589,15 @@ public class RealmsConfigurationLoader {
     }
 
     private static void consumeAttribute(JsonParser p) throws IOException {
-        JsonToken t = p.nextToken();
+        JsonToken t = p.currentToken();
         if (t == JsonToken.START_OBJECT || t == JsonToken.START_ARRAY) {
             p.skipChildren();
+            p.nextToken();
+        } else if (t == JsonToken.FIELD_NAME) {
+            p.nextToken();
+            consumeAttribute(p);
+        } else {
+            p.nextToken();
         }
     }
 
@@ -528,6 +644,40 @@ public class RealmsConfigurationLoader {
         }
     }
 
+    static class FetchMissingClientsJob extends AdminJob {
+
+        private RealmRepresentation realm;
+
+        FetchMissingClientsJob(RealmRepresentation r) {
+            realm = r;
+        }
+
+        @Override
+        public void run() {
+            List<ClientRepresentation> clients = admin().realms().realm(realm.getRealm()).clients().findAll();
+            for (ClientRepresentation c: clients) {
+                clientIdMap.put(c.getClientId(), c.getId());
+            }
+        }
+    }
+
+    static class FetchRealmRolesJob extends AdminJob {
+
+        private RealmRepresentation realm;
+
+        FetchRealmRolesJob(RealmRepresentation r) {
+            realm = r;
+        }
+
+        @Override
+        public void run() {
+            List<RoleRepresentation> roles = admin().realms().realm(realm.getRealm()).roles().list();
+            for (RoleRepresentation r: roles) {
+                realmRoleIdMap.put(r.getName(), r.getId());
+            }
+        }
+    }
+
     static class CreateRealmJob extends AdminJob {
 
         private RealmRepresentation realm;
@@ -538,7 +688,15 @@ public class RealmsConfigurationLoader {
 
         @Override
         public void run() {
-            admin().realms().create(realm);
+            try {
+                admin().realms().create(realm);
+            } catch (ClientErrorException e) {
+                if (e.getMessage().endsWith("409 Conflict") && ignoreConflicts) {
+                    log.warn("Ignoring conflict when creating a realm: " + realm.getRealm());
+                    return;
+                }
+                throw e;
+            }
         }
     }
 
@@ -556,11 +714,17 @@ public class RealmsConfigurationLoader {
         public void run() {
             Response response = admin().realms().realm(realm.getRealm()).users().create(user);
             response.close();
-            if (response.getStatus() != 201) {
+
-                throw new RuntimeException("Failed to create user with status: " + response.getStatusInfo().getReasonPhrase());
+            if (response.getStatus() == 409 && ignoreConflicts) {
+                log.warn("Ignoring conflict when creating a user: " + user.getUsername());
+                user.setId(admin().realms().realm(realm.getRealm()).users().search(user.getUsername()).get(0).getId());
+            } else if (response.getStatus() == 201) {
+                user.setId(extractIdFromResponse(response));
+            } else {
+                throw new RuntimeException("Failed to create user with status: " + response.getStatusInfo());
             }
 
-            String userId = extractIdFromResponse(response);
+            String userId = user.getId();
 
             List<CredentialRepresentation> creds = user.getCredentials();
             for (CredentialRepresentation cred: creds) {
@@ -641,8 +805,16 @@ public class RealmsConfigurationLoader {
 
         @Override
         public void run() {
-            admin().realms().realm(realm.getRealm()).roles().create(role);
+            try {
-
+                admin().realms().realm(realm.getRealm()).roles().create(role);
+            } catch (ClientErrorException e) {
+                if (e.getMessage().endsWith("409 Conflict") && ignoreConflicts) {
+                    log.warn("Ignoring conflict when creating a realm role: " + role.getName());
+                    role = admin().realms().realm(realm.getRealm()).roles().get(role.getName()).toRepresentation();
+                } else {
+                    throw e;
+                }
+            }
             // we need the id but it's not returned by REST API - we have to perform a get on the created role and save the returned id
             RoleRepresentation rr = admin().realms().realm(realm.getRealm()).roles().get(role.getName()).toRepresentation();
             realmRoleIdMap.put(rr.getName(), rr.getId());
@@ -668,7 +840,18 @@ public class RealmsConfigurationLoader {
             if (id == null) {
                 throw new RuntimeException("No client created for clientId: " + clientId);
             }
-            admin().realms().realm(realm.getRealm()).clients().get(id).roles().create(role);
+
+            try {
+                admin().realms().realm(realm.getRealm()).clients().get(id).roles().create(role);
+
+            } catch (ClientErrorException e) {
+                if (e.getMessage().endsWith("409 Conflict") && ignoreConflicts) {
+                    log.warn("Ignoring conflict when creating a client role: " + role.getName());
+                    role = admin().realms().realm(realm.getRealm()).clients().get(id).roles().get(role.getName()).toRepresentation();
+                } else {
+                    throw e;
+                }
+            }
 
             // we need the id but it's not returned by REST API - we have to perform a get on the created role and save the returned id
             RoleRepresentation rr = admin().realms().realm(realm.getRealm()).clients().get(id).roles().get(role.getName()).toRepresentation();
@@ -680,6 +863,7 @@ public class RealmsConfigurationLoader {
 
             roleIdMap.put(rr.getName(), rr.getId());
         }
+
     }
 
     static class CreateClientJob extends AdminJob {
@@ -697,11 +881,16 @@ public class RealmsConfigurationLoader {
         public void run() {
             Response response = admin().realms().realm(realm.getRealm()).clients().create(client);
             response.close();
-            if (response.getStatus() != 201) {
+
+            if (response.getStatus() == 409 && ignoreConflicts) {
+                log.warn("Ignoring conflict when creating a client: " + client.getClientId());
+                client = admin().realms().realm(realm.getRealm()).clients().findByClientId(client.getClientId()).get(0);
+            } else if (response.getStatus() == 201) {
+                client.setId(extractIdFromResponse(response));
+            } else {
                 throw new RuntimeException("Failed to create client with status: " + response.getStatusInfo().getReasonPhrase());
             }
-            String id = extractIdFromResponse(response);
+            clientIdMap.put(client.getClientId(), client.getId());
-            clientIdMap.put(client.getClientId(), id);
         }
     }
 

testsuite/performance/tests/src/main/java/org/keycloak/performance/TestConfig.java

@@ -34,6 +34,11 @@ public class TestConfig {
     // Settings used by RealmsConfigurationLoader only - when loading data into Keycloak
     //
     public static final int numOfWorkers = Integer.getInteger("numOfWorkers", 1);
+    public static final int startAtRealmIdx = Integer.getInteger("startAtRealmIdx", 0);
+    public static final int startAtUserIdx = 0; // doesn't work properly, will be removed later //Integer.getInteger("startAtUserIdx", 0);
+    public static final boolean ignoreConflicts = "true".equals(System.getProperty("ignoreConflicts", "false"));
+    public static final boolean skipRealmRoles = "true".equals(System.getProperty("skipRealmRoles", "false"));
+    public static final boolean skipClientRoles = "true".equals(System.getProperty("skipClientRoles", "false"));
 
     //
     // Settings used by RealmConfigurationLoader to connect to Admin REST API

testsuite/performance/tests/src/main/resources/logback.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+        <resetJUL>true</resetJUL>
+    </contextListener>
+
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern>
+        </encoder>
+        <immediateFlush>false</immediateFlush>
+    </appender>
+
+    <!-- Uncomment for logging ALL HTTP request and responses -->
+    <!-- 	<logger name="io.gatling.http" level="TRACE" /> -->
+    <!-- Uncomment for logging ONLY FAILED HTTP request and responses -->
+    <!-- 	<logger name="io.gatling.http" level="DEBUG" /> -->
+
+    <root level="WARN">
+        <appender-ref ref="CONSOLE" />
+    </root>
+
+</configuration>