Break up test-framework into multiple modules, and introduce placeholder for new testsuite (#34507)

Closes #34194

Signed-off-by: stianst <stianst@gmail.com>

.github/actions/conditional/action.yml

@@ -22,9 +22,6 @@ outputs:
   ci-webauthn:
     description: Should "ci.yml" execute (WebAuthn)
     value: ${{ steps.changes.outputs.ci-webauthn }}
-  ci-test-poc:
-    description: Should "ci.yml" execute (Test PoC)
-    value: ${{ steps.changes.outputs.ci-test-poc }}
   operator:
     description: Should "operator-ci.yml" execute
     value: ${{ steps.changes.outputs.operator }}

.github/actions/conditional/conditions

@@ -60,5 +60,3 @@ js/libs/keycloak-js/                        ci ci-quarkus
 *.tsx                                       codeql-typescript
 
 testsuite::database-suite                   ci-store
-
-test-poc/                                   ci ci-test-poc

.github/workflows/ci.yml

@@ -33,7 +33,6 @@ jobs:
       ci-store: ${{ steps.conditional.outputs.ci-store }}
       ci-sssd: ${{ steps.conditional.outputs.ci-sssd }}
       ci-webauthn: ${{ steps.conditional.outputs.ci-webauthn }}
-      ci-test-poc: ${{ steps.conditional.outputs.ci-test-poc }}
       ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }}
 
     steps:
@@ -84,7 +83,7 @@ jobs:
         run: |
           SEP=""
           PROJECTS=""
-          for i in `find -name '*Test.java' -type f | egrep -v './(testsuite|quarkus|docs|test-poc|test-framework)/' | sed 's|/src/test/java/.*||' | sort | uniq | sed 's|./||'`; do
+          for i in `find -name '*Test.java' -type f | egrep -v './(testsuite|quarkus|docs|tests|test-framework)/' | sed 's|/src/test/java/.*||' | sort | uniq | sed 's|./||'`; do
             PROJECTS="$PROJECTS$SEP$i"
             SEP=","
           done
@@ -958,7 +957,7 @@ jobs:
           job-id: migration-tests-${{ matrix.old-version }}-${{ matrix.database }}
 
   test-framework:
-    name: Keycloak Test Framework
+    name: Test Framework
     runs-on: ubuntu-latest
     needs: build
     timeout-minutes: 30
@@ -972,12 +971,10 @@ jobs:
       - name: Run tests
         run: ./mvnw test -f test-framework/pom.xml
 
-  test-poc:
+  base-new-integration-tests:
-    name: Test PoC
+    name: Base IT (new)
     runs-on: ubuntu-latest
-    if: needs.conditional.outputs.ci-test-poc == 'true'
     needs:
-      - conditional
       - build
     timeout-minutes: 30
     steps:
@@ -988,9 +985,7 @@ jobs:
         uses: ./.github/actions/integration-test-setup
 
       - name: Run tests
-        env:
+        run: ./mvnw test -f tests/pom.xml
-          KC_TEST_BROWSER: chrome-headless
-        run: ./mvnw clean install -f test-poc/pom.xml
 
   check:
     name: Status Check - Keycloak CI
@@ -1015,7 +1010,8 @@ jobs:
       - sssd-unit-tests
       - migration-tests
       - external-infinispan-tests
-      - test-poc
+      - test-framework
+      - base-new-integration-tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

pom.xml

@@ -307,6 +307,7 @@
         <module>authz</module>
         <module>js</module>
         <module>test-framework</module>
+        <module>tests</module>
         <module>quarkus</module>
     </modules>
 
@@ -1587,13 +1588,6 @@
             </modules>
         </profile>
 
-        <profile>
-            <id>test-poc</id>
-            <modules>
-                <module>test-poc</module>
-            </modules>
-        </profile>
-
         <!-- Profile to activate EAP8 Adapters Build -->
         <profile>
             <id>eap8-adapters</id>

test-framework/core/pom.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <parent>
+        <artifactId>keycloak-test-framework-parent</artifactId>
+        <groupId>org.keycloak.test</groupId>
+        <version>999.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>keycloak-test-framework-core</artifactId>
+    <name>Keycloak Test Framework</name>
+    <packaging>jar</packaging>
+    <description>Preview Keycloak testing framework based on JUnit 5</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-admin-client-tests</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.keycloak</groupId>
+            <artifactId>keycloak-junit5</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mariadb</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mysql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mssqlserver</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>oracle-free</artifactId>
+        </dependency>
+        <!-- Temporary dependency until we figure out how we want to support OAuth -->
+        <dependency>
+            <groupId>com.nimbusds</groupId>
+            <artifactId>oauth2-oidc-sdk</artifactId>
+            <version>11.13</version>
+        </dependency>
+    </dependencies>
+</project>

test-framework/core/src/main/java/org/keycloak/test/framework/CoreTestFrameworkExtension.java

@@ -0,0 +1,60 @@
+package org.keycloak.test.framework;
+
+import org.keycloak.test.framework.admin.KeycloakAdminClientSupplier;
+import org.keycloak.test.framework.database.DevFileDatabaseSupplier;
+import org.keycloak.test.framework.database.DevMemDatabaseSupplier;
+import org.keycloak.test.framework.database.MSSQLServerDatabaseSupplier;
+import org.keycloak.test.framework.database.MariaDBDatabaseSupplier;
+import org.keycloak.test.framework.database.MySQLDatabaseSupplier;
+import org.keycloak.test.framework.database.OracleDatabaseSupplier;
+import org.keycloak.test.framework.database.PostgresDatabaseSupplier;
+import org.keycloak.test.framework.database.TestDatabase;
+import org.keycloak.test.framework.events.EventsSupplier;
+import org.keycloak.test.framework.events.SysLogServerSupplier;
+import org.keycloak.test.framework.injection.Supplier;
+import org.keycloak.test.framework.oauth.OAuthClientSupplier;
+import org.keycloak.test.framework.realm.ClientSupplier;
+import org.keycloak.test.framework.realm.RealmSupplier;
+import org.keycloak.test.framework.realm.UserSupplier;
+import org.keycloak.test.framework.server.DistributionKeycloakTestServerSupplier;
+import org.keycloak.test.framework.server.EmbeddedKeycloakTestServerSupplier;
+import org.keycloak.test.framework.server.KeycloakTestServer;
+import org.keycloak.test.framework.server.RemoteKeycloakTestServerSupplier;
+
+import java.util.List;
+import java.util.Map;
+
+public class CoreTestFrameworkExtension implements TestFrameworkExtension {
+
+    @Override
+    public List<Supplier<?, ?>> suppliers() {
+        return List.of(
+                new KeycloakAdminClientSupplier(),
+                new ClientSupplier(),
+                new RealmSupplier(),
+                new UserSupplier(),
+                new DistributionKeycloakTestServerSupplier(),
+                new EmbeddedKeycloakTestServerSupplier(),
+                new RemoteKeycloakTestServerSupplier(),
+                new DevMemDatabaseSupplier(),
+                new DevFileDatabaseSupplier(),
+                new MySQLDatabaseSupplier(),
+                new PostgresDatabaseSupplier(),
+                new MariaDBDatabaseSupplier(),
+                new MSSQLServerDatabaseSupplier(),
+                new OracleDatabaseSupplier(),
+                new OAuthClientSupplier(),
+                new SysLogServerSupplier(),
+                new EventsSupplier()
+        );
+    }
+
+    @Override
+    public Map<Class<?>, String> valueTypeAliases() {
+        return Map.of(
+                KeycloakTestServer.class, "server",
+                TestDatabase.class, "database"
+        );
+    }
+
+}

test-framework/core/src/main/java/org/keycloak/test/framework/TestFrameworkExtension.java

@@ -0,0 +1,17 @@
+package org.keycloak.test.framework;
+
+import org.keycloak.test.framework.injection.Supplier;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public interface TestFrameworkExtension {
+
+    List<Supplier<?, ?>> suppliers();
+
+    default Map<Class<?>, String> valueTypeAliases() {
+        return Collections.emptyMap();
+    }
+
+}

test-framework/core/src/main/java/org/keycloak/test/framework/config/Config.java

@@ -13,8 +13,8 @@ public class Config {
 
     private static final SmallRyeConfig config = initConfig();
 
-    public static String getSelectedSupplier(Class<?> valueType) {
+    public static String getSelectedSupplier(Class<?> valueType, ValueTypeAlias valueTypeAlias) {
-        return config.getOptionalValue("kc.test." + ValueTypeAlias.getAlias(valueType), String.class).orElse(null);
+        return config.getOptionalValue("kc.test." + valueTypeAlias.getAlias(valueType), String.class).orElse(null);
     }
 
     public static String getAdminClientId() {

test-framework/core/src/main/java/org/keycloak/test/framework/injection/Registry.java

@@ -2,9 +2,8 @@ package org.keycloak.test.framework.injection;
 
 import org.jboss.logging.Logger;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.keycloak.test.framework.TestFrameworkExtension;
 import org.keycloak.test.framework.config.Config;
-import org.keycloak.test.framework.realm.DefaultRealmConfig;
-import org.keycloak.test.framework.realm.ManagedRealm;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Field;
@@ -288,17 +287,24 @@ public class Registry implements ExtensionContext.Store.CloseableResource {
     }
 
     private void loadSuppliers() {
-        Iterator<Supplier> supplierIterator = ServiceLoader.load(Supplier.class).iterator();
+        Iterator<TestFrameworkExtension> extensions = ServiceLoader.load(TestFrameworkExtension.class).iterator();
+        ValueTypeAlias valueTypeAlias = new ValueTypeAlias();
+        List<Supplier> tmp = new LinkedList<>();
+        while (extensions.hasNext()) {
+            TestFrameworkExtension extension = extensions.next();
+            tmp.addAll(extension.suppliers());
+            valueTypeAlias.addAll(extension.valueTypeAliases());
+        }
+
         Set<Class> loadedValueTypes = new HashSet<>();
         Set<Supplier> skippedSuppliers = new HashSet<>();
 
-        while (supplierIterator.hasNext()) {
+        for (Supplier supplier : tmp) {
-            Supplier supplier = supplierIterator.next();
             boolean shouldAdd = false;
             Class supplierValueType = supplier.getValueType();
 
             if (!loadedValueTypes.contains(supplierValueType)) {
-                String requestedSupplier = Config.getSelectedSupplier(supplierValueType);
+                String requestedSupplier = Config.getSelectedSupplier(supplierValueType, valueTypeAlias);
                 if (requestedSupplier != null) {
                     if (requestedSupplier.equals(supplier.getAlias())) {
                         shouldAdd = true;
@@ -320,14 +326,14 @@ public class Registry implements ExtensionContext.Store.CloseableResource {
             StringBuilder loaded = new StringBuilder();
             loaded.append("Loaded suppliers:");
             for (Supplier s : suppliers) {
-                loaded.append("\n - " + ValueTypeAlias.getAlias(s.getValueType()) + " --> " + s.getAlias());
+                loaded.append("\n - " + valueTypeAlias.getAlias(s.getValueType()) + " --> " + s.getAlias());
             }
             LOGGER.trace(loaded.toString());
 
             StringBuilder skipped = new StringBuilder();
             skipped.append("Skipped suppliers:");
             for (Supplier s : skippedSuppliers) {
-                skipped.append("\n - " + ValueTypeAlias.getAlias(s.getValueType()) + " --> " + s.getAlias());
+                skipped.append("\n - " + valueTypeAlias.getAlias(s.getValueType()) + " --> " + s.getAlias());
             }
             LOGGER.trace(skipped.toString());
         }

test-framework/core/src/main/java/org/keycloak/test/framework/injection/ValueTypeAlias.java

@@ -0,0 +1,22 @@
+package org.keycloak.test.framework.injection;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ValueTypeAlias {
+
+    private final Map<Class<?>, String> aliases = new HashMap<>();
+
+    public void addAll(Map<Class<?>, String> aliases) {
+        this.aliases.putAll(aliases);
+    }
+
+    public String getAlias(Class<?> clazz) {
+        String alias = aliases.get(clazz);
+        if (alias == null) {
+            alias = clazz.getSimpleName();
+        }
+        return alias;
+    }
+
+}

test-framework/core/src/main/resources/META-INF/services/org.keycloak.test.framework.TestFrameworkExtension

@@ -0,0 +1 @@
+org.keycloak.test.framework.CoreTestFrameworkExtension

test-framework/core/src/test/java/org/keycloak/test/framework/injection/ValueTypeAliasTest.java

@@ -0,0 +1,24 @@
+package org.keycloak.test.framework.injection;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.test.framework.server.KeycloakTestServer;
+
+import java.util.Map;
+
+public class ValueTypeAliasTest {
+
+    @Test
+    public void withAlias() {
+        ValueTypeAlias valueTypeAlias = new ValueTypeAlias();
+        valueTypeAlias.addAll(Map.of(KeycloakTestServer.class, "server"));
+        Assertions.assertEquals("server", valueTypeAlias.getAlias(KeycloakTestServer.class));
+    }
+
+    @Test
+    public void withoutAlias() {
+        Assertions.assertEquals("Keycloak", new ValueTypeAlias().getAlias(Keycloak.class));
+    }
+
+}

test-framework/core/src/test/java/org/keycloak/test/framework/injection/mocks/MockTestFrameworkExtension.java

@@ -0,0 +1,19 @@
+package org.keycloak.test.framework.injection.mocks;
+
+import org.keycloak.test.framework.TestFrameworkExtension;
+import org.keycloak.test.framework.injection.Supplier;
+
+import java.util.List;
+
+public class MockTestFrameworkExtension implements TestFrameworkExtension {
+
+    @Override
+    public List<Supplier<?, ?>> suppliers() {
+        return List.of(
+                new MockParentSupplier(),
+                new MockParent2Supplier(),
+                new MockChildSupplier()
+        );
+    }
+
+}

test-framework/core/src/test/resources/META-INF/services/org.keycloak.test.framework.TestFrameworkExtension

@@ -0,0 +1 @@
+org.keycloak.test.framework.injection.mocks.MockTestFrameworkExtension

test-framework/examples/pom.xml

@@ -19,23 +19,29 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <parent>
-        <artifactId>keycloak-test-parent</artifactId>
+        <artifactId>keycloak-test-framework-parent</artifactId>
         <groupId>org.keycloak.test</groupId>
         <version>999.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>keycloak-test-framework-showcase</artifactId>
+    <artifactId>keycloak-test-framework-examples</artifactId>
-    <name>Keycloak Test Framework Showcase</name>
+    <name>Keycloak Test Framework Examples</name>
     <packaging>jar</packaging>
-    <description>Showcase of new tests demonstrating the new testing framework usability</description>
+    <description>Example tests to demonstrate the new testing framework</description>
 
     <dependencies>
         <dependency>
-            <groupId>org.keycloak</groupId>
+            <groupId>org.keycloak.test</groupId>
-            <artifactId>keycloak-test-framework</artifactId>
+            <artifactId>keycloak-test-framework-core</artifactId>
-            <version>999.0.0-SNAPSHOT</version>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.keycloak.test</groupId>
+            <artifactId>keycloak-test-framework-ui</artifactId>
+            <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -53,6 +59,10 @@
             <groupId>com.microsoft.sqlserver</groupId>
             <artifactId>mssql-jdbc</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.oracle.database.jdbc</groupId>
+            <artifactId>ojdbc11</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -66,6 +76,13 @@
                     </systemPropertyVariables>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>