KEYCLOAK-17412 Improve control of model tests

core/src/main/java/org/keycloak/Config.java

@@ -75,7 +75,7 @@ public class Config {
 
     public static class SystemPropertiesScope implements Scope {
 
-        private String prefix;
+        protected String prefix;
 
         public SystemPropertiesScope(String prefix) {
             this.prefix = prefix;

testsuite/model/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <scope>test</scope>
+            <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>org.hamcrest</groupId>
@@ -111,6 +111,26 @@
                     </systemPropertyVariables>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <delete>
+                                    <fileset dir="${project.build.directory}" includes="map-*.json"/>
+                                    <fileset dir="${project.build.directory}" includes="map/**/*.json"/>
+                                </delete>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>                        
         </plugins>
     </build>
     
@@ -174,7 +194,14 @@
         <profile>
             <id>map+infinispan</id>
             <properties>
-                <keycloak.model.parameters>Infinispan,Map,ConcurrentHashMapStorage</keycloak.model.parameters>
+                <keycloak.model.parameters>Infinispan,Jpa,Map,ConcurrentHashMapStorage</keycloak.model.parameters>
+            </properties>
+        </profile>
+
+        <profile>
+            <id>map</id>
+            <properties>
+                <keycloak.model.parameters>Jpa,Map,ConcurrentHashMapStorage</keycloak.model.parameters>
             </properties>
         </profile>
     </profiles>

testsuite/model/src/main/java/org/keycloak/testsuite/model/Config.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.model;
+
+import org.keycloak.Config.ConfigProvider;
+import org.keycloak.Config.Scope;
+import org.keycloak.Config.SystemPropertiesScope;
+import org.keycloak.common.util.StringPropertyReplacer;
+import org.keycloak.common.util.SystemEnvProperties;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class Config implements ConfigProvider {
+
+    private final Properties systemProperties = new SystemEnvProperties();
+
+    private final Map<String, String> properties = new HashMap<>();
+
+    public class SpiConfig {
+
+        private final String prefix;
+
+        public SpiConfig(String prefix) {
+            this.prefix = prefix;
+        }
+
+        public ProviderConfig provider(String provider) {
+            return new ProviderConfig(this, prefix + provider + ".");
+        }
+
+        public SpiConfig defaultProvider(String defaultProviderId) {
+            return config("provider", defaultProviderId);
+        }
+
+        public SpiConfig config(String key, String value) {
+            if (value == null) {
+                properties.remove(prefix + key);
+            } else {
+                properties.put(prefix + key, value);
+            }
+            return this;
+        }
+
+        public SpiConfig spi(String spiName) {
+            return new SpiConfig(spiName + ".");
+        }
+    }
+
+    public class ProviderConfig {
+
+        private final SpiConfig spiConfig;
+        private final String prefix;
+
+        public ProviderConfig(SpiConfig spiConfig, String prefix) {
+            this.spiConfig = spiConfig;
+            this.prefix = prefix;
+        }
+
+        public ProviderConfig config(String key, String value) {
+            if (value == null) {
+                properties.remove(prefix + key);
+            } else {
+                properties.put(prefix + key, value);
+            }
+            return this;
+        }
+
+        public ProviderConfig provider(String provider) {
+            return spiConfig.provider(provider);
+        }
+
+        public SpiConfig spi(String spiName) {
+            return new SpiConfig(spiName + ".");
+        }
+
+    }
+
+    private class MapConfigScope extends SystemPropertiesScope {
+
+        public MapConfigScope(String prefix) {
+            super(prefix);
+        }
+
+        @Override
+        public String get(String key, String defaultValue) {
+            String v = replaceProperties(properties.get(prefix + key));
+            if (v == null || v.isEmpty()) {
+                v = System.getProperty("keycloak." + prefix + key, defaultValue);
+            }
+            return v != null && ! v.isEmpty() ? v : null;
+        }
+
+        @Override
+        public Scope scope(String... scope) {
+            StringBuilder sb = new StringBuilder();
+            sb.append(prefix).append(".");
+            for (String s : scope) {
+                sb.append(s);
+                sb.append(".");
+            }
+            return new MapConfigScope(sb.toString());
+        }
+    }
+
+    @Override
+    public String getProvider(String spiName) {
+        return properties.get(spiName + ".provider");
+    }
+
+    private String replaceProperties(String value) {
+        return StringPropertyReplacer.replaceProperties(value, systemProperties);
+    }
+
+    @Override
+    public Scope scope(String... scope) {
+        StringBuilder sb = new StringBuilder();
+        for (String s : scope) {
+            sb.append(s);
+            sb.append(".");
+        }
+        return new MapConfigScope(sb.toString());
+    }
+
+    public SpiConfig spi(String spiName) {
+        return new SpiConfig(spiName + ".");
+    }
+
+    @Override
+    public String toString() {
+        return properties.entrySet().stream()
+          .sorted((e1, e2) -> e1.getKey().compareTo(e2.getKey()))
+          .map(e -> e.getKey() + " = " + e.getValue())
+          .collect(Collectors.joining("\n    "));
+    }
+}

testsuite/model/src/main/java/org/keycloak/testsuite/model/KeycloakModelParameters.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2020 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.model;
+
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
+import java.util.Set;
+import java.util.stream.Stream;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class KeycloakModelParameters {
+
+    private final Set<Class<? extends Spi>> allowedSpis;
+    private final Set<Class<? extends ProviderFactory>> allowedFactories;
+
+    public KeycloakModelParameters(Set<Class<? extends Spi>> allowedSpis, Set<Class<? extends ProviderFactory>> allowedFactories) {
+        this.allowedSpis = allowedSpis;
+        this.allowedFactories = allowedFactories;
+    }
+
+    boolean isSpiAllowed(Spi s) {
+        return allowedSpis.contains(s.getClass());
+    }
+
+    boolean isFactoryAllowed(ProviderFactory factory) {
+        return allowedFactories.stream().anyMatch((c) -> c.isAssignableFrom(factory.getClass()));
+    }
+
+    /**
+     * Returns stream of parameters of the given type, or an empty stream if no parameters of the given type are supplied
+     * by this clazz.
+     * @param <T>
+     * @param clazz
+     * @return
+     */
+    public <T> Stream<T> getParameters(Class<T> clazz) {
+        return Stream.empty();
+    }
+
+    public Statement classRule(Statement base, Description description) {
+        return base;
+    }
+
+    public Statement instanceRule(Statement base, Description description) {
+        return base;
+    }
+
+}

testsuite/model/src/main/java/org/keycloak/testsuite/model/RequireProvider.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.model;
+
+import org.keycloak.provider.Provider;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Identifies a requirement for a given provider to be present in the session factory.
+ * If the provider is not available, the test is skipped.
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Repeatable(RequireProviders.class)
+public @interface RequireProvider {
+    Class<? extends Provider> value() default Provider.class;
+
+}

testsuite/model/src/main/java/org/keycloak/testsuite/model/RequireProviders.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.model;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface RequireProviders {
+    RequireProvider[] value();
+}

testsuite/model/src/test/java/org/keycloak/testsuite/model/ClientScopeStorageTest.java

@@ -61,6 +61,7 @@ public class ClientScopeStorageTest extends KeycloakModelTest {
             ComponentModel res = realm.addComponentModel(federatedStorage);
             clientScopeFederationId = res.getId();
             log.infof("Added %s client scope federation provider: %s", federatedStorage.getName(), clientScopeFederationId);
+            return null;
         }));
 
         inComittedTransaction(1, (session, i) -> {
@@ -68,6 +69,7 @@ public class ClientScopeStorageTest extends KeycloakModelTest {
             StorageId storageId = new StorageId(clientScopeFederationId, "scope_name");
             ClientScopeModel hardcoded = session.clientScopes().getClientScopeById(realm, storageId.getId());
             Assert.assertNotNull(hardcoded);
+            return null;
         });
     }
 }

testsuite/model/src/test/java/org/keycloak/testsuite/model/KeycloakModelParameters.java

@@ -18,6 +18,10 @@ package org.keycloak.testsuite.model;
 
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.provider.Spi;
+import org.keycloak.testsuite.model.Config.SpiConfig;
+import org.keycloak.util.JsonSerialization;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.util.Set;
 import java.util.stream.Stream;
 import org.junit.runner.Description;
@@ -56,6 +60,9 @@ public class KeycloakModelParameters {
         return Stream.empty();
     }
 
+    public void updateConfig(Config cf) {
+    }
+
     public Statement classRule(Statement base, Description description) {
         return base;
     }

testsuite/model/src/test/java/org/keycloak/testsuite/model/KeycloakModelTest.java

@@ -30,6 +30,7 @@ import org.keycloak.models.ClientSpi;
 import org.keycloak.models.GroupSpi;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.RealmModel;
 import org.keycloak.models.RealmSpi;
 import org.keycloak.models.RoleSpi;
 import org.keycloak.models.UserSpi;
@@ -47,8 +48,11 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.hamcrest.Matchers;
@@ -92,8 +96,11 @@ public abstract class KeycloakModelTest {
                 st = Stream.concat(Stream.of(testClass.getAnnotationsByType(RequireProvider.class)), st);
                 testClass = testClass.getSuperclass();
             }
-            List<Class<? extends Provider>> notFound = st.map(RequireProvider::value)
+            List<Class<? extends Provider>> notFound = st
-              .filter(pClass -> FACTORY.getProviderFactory(pClass) == null)
+              .filter(rp -> rp.only().length == 0 
+                ? FACTORY.getProviderFactory(rp.value()) == null
+                : Stream.of(rp.only()).allMatch(provider -> FACTORY.getProviderFactory(rp.value(), provider) == null))
+              .map(RequireProvider::value)
               .collect(Collectors.toList());
             Assume.assumeThat("Some required providers not found", notFound, Matchers.empty());
 
@@ -119,16 +126,30 @@ public abstract class KeycloakModelTest {
                 st = Stream.concat(st, Stream.of(rp));
             }
 
-            for (Iterator<Class<? extends Provider>> iterator = st.map(RequireProvider::value).iterator(); iterator.hasNext();) {
+            for (Iterator<RequireProvider> iterator = st.iterator(); iterator.hasNext();) {
-                Class<? extends Provider> providerClass = iterator.next();
+                RequireProvider rpInner = iterator.next();
+                Class<? extends Provider> providerClass = rpInner.value();
+                String[] only = rpInner.only();
 
-                if (FACTORY.getProviderFactory(providerClass) == null) {
+                if (only.length == 0) {
-                    return new Statement() {
+                    if (FACTORY.getProviderFactory(providerClass) == null) {
-                        @Override
+                        return new Statement() {
-                        public void evaluate() throws Throwable {
+                            @Override
-                            throw new AssumptionViolatedException("Provider must exist: " + providerClass);
+                            public void evaluate() throws Throwable {
-                        }
+                                throw new AssumptionViolatedException("Provider must exist: " + providerClass);
-                    };
+                            }
+                        };
+                    }
+                } else {
+                    boolean notFoundAny = Stream.of(only).allMatch(provider -> FACTORY.getProviderFactory(providerClass, provider) == null);
+                    if (notFoundAny) {
+                        return new Statement() {
+                            @Override
+                            public void evaluate() throws Throwable {
+                                throw new AssumptionViolatedException("Provider must exist: " + providerClass + " one of [" + String.join(",", only) + "]");
+                            }
+                        };
+                    }
                 }
             }
 
@@ -160,7 +181,8 @@ public abstract class KeycloakModelTest {
       .build();
 
     protected static final List<KeycloakModelParameters> MODEL_PARAMETERS;
-    protected static final DefaultKeycloakSessionFactory FACTORY;
+    protected static Config CONFIG;
+    protected static DefaultKeycloakSessionFactory FACTORY;
 
     static {
         KeycloakModelParameters basicParameters = new KeycloakModelParameters(ALLOWED_SPIS, ALLOWED_FACTORIES);
@@ -170,13 +192,22 @@ public abstract class KeycloakModelTest {
             .filter(s -> s != null && ! s.trim().isEmpty())
             .map(cn -> { try { return Class.forName(cn.indexOf('.') >= 0 ? cn : ("org.keycloak.testsuite.model.parameters." + cn)); } catch (Exception e) { LOG.error("Cannot find " + cn); return null; }})
             .filter(Objects::nonNull)
-            .map(c -> { try { return c.newInstance(); } catch (Exception e) { LOG.error("Cannot instantiate " + c); return null; }} )
+            .map(c -> { try { return c.getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Cannot instantiate " + c); return null; }} )
             .filter(KeycloakModelParameters.class::isInstance)
             .map(KeycloakModelParameters.class::cast)
           )
           .collect(Collectors.toList());
 
-        FACTORY = new DefaultKeycloakSessionFactory() {
+        reinitializeKeycloakSessionFactory();
+    }
+
+    public static DefaultKeycloakSessionFactory createKeycloakSessionFactory() {
+        CONFIG = new Config();
+        MODEL_PARAMETERS.forEach(m -> m.updateConfig(CONFIG));
+        LOG.debug("Using the following configuration:\n    " + CONFIG);
+        org.keycloak.Config.init(CONFIG);
+
+        DefaultKeycloakSessionFactory res = new DefaultKeycloakSessionFactory() {
             @Override
             protected boolean isEnabled(ProviderFactory factory, Scope scope) {
                 return super.isEnabled(factory, scope) && isFactoryAllowed(factory);
@@ -196,7 +227,16 @@ public abstract class KeycloakModelTest {
                 return MODEL_PARAMETERS.stream().anyMatch(p -> p.isFactoryAllowed(factory));
             }
         };
-        FACTORY.init();
+        res.init();
+        return res;
+    }
+
+    public static void reinitializeKeycloakSessionFactory() {
+        DefaultKeycloakSessionFactory f = createKeycloakSessionFactory();
+        if (FACTORY != null) {
+            FACTORY.close();
+        }
+        FACTORY = f;
     }
 
     @BeforeClass
@@ -237,11 +277,20 @@ public abstract class KeycloakModelTest {
         session.getTransactionManager().rollback();
     }
 
-    protected <T> void inComittedTransaction(T parameter, BiConsumer<KeycloakSession, T> what) {
+    protected <T, R> R inComittedTransaction(T parameter, BiFunction<KeycloakSession, T, R> what) {
-        inComittedTransaction(parameter, what, (a,b) -> {}, (a,b) -> {});
+        return inComittedTransaction(parameter, what, null, null);
     }
 
-    protected <T> void inComittedTransaction(T parameter, BiConsumer<KeycloakSession, T> what, BiConsumer<KeycloakSession, T> onCommit, BiConsumer<KeycloakSession, T> onRollback) {
+    protected void inComittedTransaction(Consumer<KeycloakSession> what) {
+        inComittedTransaction(a -> { what.accept(a); return null; });
+    }
+
+    protected <R> R inComittedTransaction(Function<KeycloakSession, R> what) {
+        return inComittedTransaction(1, (a,b) -> what.apply(a));
+    }
+
+    protected <T, R> R inComittedTransaction(T parameter, BiFunction<KeycloakSession, T, R> what, BiConsumer<KeycloakSession, T> onCommit, BiConsumer<KeycloakSession, T> onRollback) {
+        AtomicReference<R> res = new AtomicReference<>();
         KeycloakModelUtils.runJobInTransaction(FACTORY, session -> {
             session.getTransactionManager().enlistAfterCompletion(new AbstractKeycloakTransaction() {
                 @Override
@@ -254,7 +303,16 @@ public abstract class KeycloakModelTest {
                     if (onRollback != null) { onRollback.accept(session, parameter); }
                 }
             });
-            what.accept(session, parameter);
+            res.set(what.apply(session, parameter));
+        });
+        return res.get();
+    }
+
+    protected <R> R withRealm(String realmId, BiFunction<KeycloakSession, RealmModel, R> what) {
+        return inComittedTransaction(session -> {
+            final RealmModel realm = session.realms().getRealm(realmId);
+            session.getContext().setRealm(realm);
+            return what.apply(session, realm);
         });
     }
 

testsuite/model/src/test/java/org/keycloak/testsuite/model/RequireProvider.java

@@ -35,4 +35,10 @@ import java.lang.annotation.Target;
 public @interface RequireProvider {
     Class<? extends Provider> value() default Provider.class;
 
+    /**
+     * Specifies provider IDs of mandatory provider. There must be at least one provider available
+     * from those in {@code only} array to fulfil this requirement.
+     */
+    String[] only() default {};
+
 }

testsuite/model/src/test/java/org/keycloak/testsuite/model/UserModelTest.java

@@ -81,7 +81,7 @@ public class UserModelTest extends KeycloakModelTest {
         s.realms().removeRealm(realmId);
     }
 
-    private void addRemoveUser(KeycloakSession session, int i) {
+    private Void addRemoveUser(KeycloakSession session, int i) {
         RealmModel realm = session.realms().getRealmByName("realm");
 
         UserModel user = session.users().addUser(realm, "user-" + i);
@@ -102,6 +102,8 @@ public class UserModelTest extends KeycloakModelTest {
         assertTrue(session.users().removeUser(realm, user));
         assertFalse(session.users().removeUser(realm, user));
         assertNull(session.users().getUserByUsername(realm, user.getUsername()));
+
+        return null;
     }
 
     @Test
@@ -125,9 +127,10 @@ public class UserModelTest extends KeycloakModelTest {
             final UserModel user = session.users().addUser(realm, "user-" + i);
             user.joinGroup(session.groups().getGroupById(realm, groupId));
             userIds.add(user.getId());
+            return null;
         }));
 
-        inComittedTransaction(1, (session, i) -> {
+        inComittedTransaction(session -> {
             final RealmModel realm = session.realms().getRealm(realmId);
             final GroupModel group = session.groups().getGroupById(realm, groupId);
             assertThat(session.users().getGroupMembersStream(realm, group).count(), is(100L));
@@ -140,6 +143,7 @@ public class UserModelTest extends KeycloakModelTest {
                 final RealmModel realm = session.realms().getRealm(realmId);
                 final UserModel user = session.users().getUserById(realm, userId);
                 log.debugf("Remove user %s: %s", userId, session.users().removeUser(realm, user));
+                return null;
             }, null, (session, userId) -> remainingUserIds.add(userId) ));
 
             userIds.clear();
@@ -147,7 +151,7 @@ public class UserModelTest extends KeycloakModelTest {
             remainingUserIds.clear();
         } while (! userIds.isEmpty());
 
-        inComittedTransaction(1, (session, i) -> {
+        inComittedTransaction(session -> {
             final RealmModel realm = session.realms().getRealm(realmId);
             final GroupModel group = session.groups().getGroupById(realm, groupId);
             assertThat(session.users().getGroupMembersStream(realm, group).collect(Collectors.toList()), Matchers.empty());
@@ -159,27 +163,24 @@ public class UserModelTest extends KeycloakModelTest {
     public void testAddDirtyRemoveFederationUser() {
         registerUserFederationWithRealm();
 
-        inComittedTransaction(1, (session, i) -> {
+        withRealm(realmId, (session, realm) -> session.users().addUser(realm, "user-A"));
-            final RealmModel realm = session.realms().getRealm(realmId);
-            final UserModel user = session.users().addUser(realm, "user-A");
-        });
 
         // Remove user _from the federation_, simulates eg. user being removed from LDAP without Keycloak knowing
-        inComittedTransaction(1, (session, i) -> {
+        withRealm(realmId, (session, realm) -> {
-            final RealmModel realm = session.realms().getRealm(realmId);
             final UserStorageProvider instance = getUserFederationInstance(session, realm);
             log.debugf("Removing selected users from backend");
             final UserModel user = session.users().getUserByUsername(realm, "user-A");
             ((UserRegistrationProvider) instance).removeUser(realm, user);
+            return null;
         });
 
-        inComittedTransaction(1, (session, i) -> {
+        withRealm(realmId, (session, realm) -> {
-            final RealmModel realm = session.realms().getRealm(realmId);
             if (session.userCache() != null) {
                 session.userCache().clear();
             }
             final UserModel user = session.users().getUserByUsername(realm, "user-A");
             assertThat("User should not be found in the main store", user, Matchers.nullValue());
+            return null;
         });
     }
 
@@ -198,26 +199,27 @@ public class UserModelTest extends KeycloakModelTest {
             user.joinGroup(session.groups().getGroupById(realm, groupId));
             log.infof("Created user with id: %s", user.getId());
             userIds.add(user.getId());
+            return null;
         }));
 
         // Remove users _from the federation_, simulates eg. user being removed from LDAP without Keycloak knowing
-        inComittedTransaction(1, (session, i) -> {
+        withRealm(realmId, (session, realm) -> {
-            final RealmModel realm = session.realms().getRealm(realmId);
             UserStorageProvider instance = getUserFederationInstance(session, realm);
             log.debugf("Removing selected users from backend");
             IntStream.range(FIRST_DELETED_USER_INDEX, LAST_DELETED_USER_INDEX).forEach(j -> {
                 final UserModel user = session.users().getUserByUsername(realm, "user-" + j);
                 ((UserRegistrationProvider) instance).removeUser(realm, user);
             });
+            return null;
         });
 
-        IntStream.range(0, 7).parallel().forEach(index -> inComittedTransaction(index, (session, i) -> {
+        IntStream.range(0, 7).parallel().forEach(index -> withRealm(realmId, (session, realm) -> {
-            final RealmModel realm = session.realms().getRealm(realmId);
             final GroupModel group = session.groups().getGroupById(realm, groupId);
             assertThat(session.users().getGroupMembersStream(realm, group).count(), is(100L - DELETED_USER_COUNT));
+            return null;
         }));
 
-        inComittedTransaction(1, (session, i) -> {
+        inComittedTransaction(session -> {
             // If we are using cache, we need to invalidate all users because after removing users from external
             // provider cache may not be cleared and it may be the case, that cache is the only place that is having 
             // a reference to removed users. Our importValidation method won't be called at all for removed users
@@ -227,6 +229,7 @@ public class UserModelTest extends KeycloakModelTest {
             if (session.userCache() != null) {
                 session.userCache().clear();
             }
+            return null;
         });
 
         // Now delete the users, and count those that were not found to be deleted. This should be equal to the number
@@ -245,6 +248,7 @@ public class UserModelTest extends KeycloakModelTest {
                     log.debugf("Failed deleting user: %s", userId);
                     notFoundUsers.incrementAndGet();
                 }
+                return null;
             }, null, (session, userId) -> {
                 log.debugf("Could not delete user %s", userId);
                 remainingUserIds.add(userId);
@@ -257,22 +261,22 @@ public class UserModelTest extends KeycloakModelTest {
 
         assertThat(notFoundUsers.get(), is(DELETED_USER_COUNT));
 
-        inComittedTransaction(1, (session, i) -> {
+        withRealm(realmId, (session, realm) -> {
-            final RealmModel realm = session.realms().getRealm(realmId);
             final GroupModel group = session.groups().getGroupById(realm, groupId);
             assertThat(session.users().getGroupMembersStream(realm, group).collect(Collectors.toList()), Matchers.empty());
+            return null;
         });
     }
 
     private void registerUserFederationWithRealm() {
-        getParameters(UserStorageProviderModel.class).forEach(fs -> inComittedTransaction(fs, (session, federatedStorage) -> {
+        getParameters(UserStorageProviderModel.class).forEach(fs -> inComittedTransaction(session -> {
             assumeThat("Cannot handle more than 1 user federation provider", userFederationId, Matchers.nullValue());
             RealmModel realm = session.realms().getRealm(realmId);
-            federatedStorage.setParentId(realmId);
+            fs.setParentId(realmId);
-            federatedStorage.setImportEnabled(true);
+            fs.setImportEnabled(true);
-            ComponentModel res = realm.addComponentModel(federatedStorage);
+            ComponentModel res = realm.addComponentModel(fs);
             userFederationId = res.getId();
-            log.infof("Added %s user federation provider: %s", federatedStorage.getName(), userFederationId);
+            log.infof("Added %s user federation provider: %s", fs.getName(), userFederationId);
         }));
     }
 

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/ConcurrentHashMapStorage.java

@@ -16,15 +16,12 @@
  */
 package org.keycloak.testsuite.model.parameters;
 
-import org.keycloak.testsuite.model.KeycloakModelParameters;
-import org.keycloak.models.map.client.MapClientProviderFactory;
-import org.keycloak.models.map.group.MapGroupProviderFactory;
-import org.keycloak.models.map.role.MapRoleProviderFactory;
-import org.keycloak.models.map.storage.chm.ConcurrentHashMapStorageProvider;
-import org.keycloak.models.map.storage.MapStorageProvider;
 import org.keycloak.models.map.storage.MapStorageSpi;
+import org.keycloak.testsuite.model.KeycloakModelParameters;
+import org.keycloak.models.map.storage.chm.ConcurrentHashMapStorageProvider;
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.provider.Spi;
+import org.keycloak.testsuite.model.Config;
 import com.google.common.collect.ImmutableSet;
 import java.util.Set;
 
@@ -41,8 +38,12 @@ public class ConcurrentHashMapStorage extends KeycloakModelParameters {
       .add(ConcurrentHashMapStorageProvider.class)
       .build();
 
-    static {
+    @Override
-        System.setProperty("keycloak.mapStorage.concurrenthashmap.dir", System.getProperty("keycloak.mapStorage.concurrenthashmap.dir", "${project.build.directory:target}"));
+    public void updateConfig(Config cf) {
+        cf.spi(MapStorageSpi.NAME)
+            .defaultProvider(ConcurrentHashMapStorageProvider.PROVIDER_ID)
+            .provider(ConcurrentHashMapStorageProvider.PROVIDER_ID)
+              .config("dir", "${project.build.directory:target}");
     }
 
     public ConcurrentHashMapStorage() {

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/Infinispan.java

@@ -26,6 +26,7 @@ import org.keycloak.models.cache.infinispan.InfinispanCacheRealmProviderFactory;
 import org.keycloak.models.cache.infinispan.InfinispanUserCacheProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.provider.Spi;
+import org.keycloak.testsuite.model.Config;
 import com.google.common.collect.ImmutableSet;
 import java.util.Set;
 
@@ -49,8 +50,11 @@ public class Infinispan extends KeycloakModelParameters {
       .add(InfinispanUserCacheProviderFactory.class)
       .build();
 
-    static {
+    @Override
-        System.setProperty("keycloak.connectionsInfinispan.default.embedded", System.getProperty("keycloak.connectionsInfinispan.default.embedded", "true"));
+    public void updateConfig(Config cf) {
+        cf.spi("connectionsInfinispan")
+            .provider("default")
+              .config("embedded", "true");
     }
 
     public Infinispan() {

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/Jpa.java

@@ -35,6 +35,7 @@ import org.keycloak.models.jpa.JpaRoleProviderFactory;
 import org.keycloak.models.jpa.JpaUserProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.provider.Spi;
+import org.keycloak.testsuite.model.Config;
 import com.google.common.collect.ImmutableSet;
 import java.util.Set;
 
@@ -73,4 +74,14 @@ public class Jpa extends KeycloakModelParameters {
         super(ALLOWED_SPIS, ALLOWED_FACTORIES);
     }
 
+
+    @Override
+    public void updateConfig(Config cf) {
+        cf.spi("client").defaultProvider("jpa")
+          .spi("clientScope").defaultProvider("jpa")
+          .spi("group").defaultProvider("jpa")
+          .spi("role").defaultProvider("jpa")
+          .spi("user").defaultProvider("jpa")
+        ;
+    }
 }

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/Map.java

@@ -18,12 +18,15 @@ package org.keycloak.testsuite.model.parameters;
 
 import org.keycloak.testsuite.model.KeycloakModelParameters;
 import org.keycloak.models.map.client.MapClientProviderFactory;
+import org.keycloak.models.map.clientscope.MapClientScopeProviderFactory;
 import org.keycloak.models.map.group.MapGroupProviderFactory;
 import org.keycloak.models.map.role.MapRoleProviderFactory;
 import org.keycloak.models.map.storage.MapStorageProvider;
 import org.keycloak.models.map.storage.MapStorageSpi;
+import org.keycloak.models.map.user.MapUserProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.provider.Spi;
+import org.keycloak.testsuite.model.Config;
 import com.google.common.collect.ImmutableSet;
 import java.util.Set;
 
@@ -40,12 +43,24 @@ public class Map extends KeycloakModelParameters {
 
     static final Set<Class<? extends ProviderFactory>> ALLOWED_FACTORIES = ImmutableSet.<Class<? extends ProviderFactory>>builder()
       .add(MapClientProviderFactory.class)
+      .add(MapClientScopeProviderFactory.class)
       .add(MapGroupProviderFactory.class)
       .add(MapRoleProviderFactory.class)
+      .add(MapUserProviderFactory.class)
       .add(MapStorageProvider.class)
       .build();
 
     public Map() {
         super(ALLOWED_SPIS, ALLOWED_FACTORIES);
     }
+
+    @Override
+    public void updateConfig(Config cf) {
+        cf.spi("client").defaultProvider(MapClientProviderFactory.PROVIDER_ID)
+          .spi("clientScope").defaultProvider(MapClientScopeProviderFactory.PROVIDER_ID)
+          .spi("group").defaultProvider(MapGroupProviderFactory.PROVIDER_ID)
+          .spi("role").defaultProvider(MapRoleProviderFactory.PROVIDER_ID)
+          .spi("user").defaultProvider(MapUserProviderFactory.PROVIDER_ID)
+        ;
+    }
 }

testsuite/model/src/test/resources/log4j.properties

@@ -32,8 +32,8 @@ log4j.logger.org.keycloak.testsuite=${keycloak.testsuite.logging.level}
 # log4j.logger.org.hibernate=debug
 
 # Enable to view loaded SPI and Providers
-# log4j.logger.org.keycloak.services.DefaultKeycloakSessionFactory=debug
+ log4j.logger.org.keycloak.services.DefaultKeycloakSessionFactory=debug
-# log4j.logger.org.keycloak.provider.ProviderManager=debug
+ log4j.logger.org.keycloak.provider.ProviderManager=debug
 # log4j.logger.org.keycloak.provider.FileSystemProviderLoaderFactory=debug
 
 # Liquibase updates logged with "info" by default. Logging level can be changed by system property "keycloak.liquibase.logging.level"

testsuite/model/test-all-profiles.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 cd "$(dirname $0)"
+mvn -version
 
 EXIT_CODE=0
 mvn clean