Database configuration tests

Closes #9293

.github/workflows/ci.yml

@@ -425,6 +425,13 @@ jobs:
           find . -path '*/target/surefire-reports/*.xml' | zip -q reports-quarkus-tests.zip -@
           exit $TEST_RESULT
 
+      - name: Run Quarkus Storage Tests
+        run: |
+          mvn clean install -nsu -B -f quarkus/tests/pom.xml -Ptest-database -Dtest=PostgreSQLStartDatabaseTest | misc/log/trimmer.sh
+          TEST_RESULT=${PIPESTATUS[0]}
+          find . -path '*/target/surefire-reports/*.xml' | zip -q reports-quarkus-tests.zip -@
+          exit $TEST_RESULT
+
       - name: Quarkus test reports
         uses: actions/upload-artifact@v2
         if: failure()

quarkus/deployment/src/main/java/org/keycloak/quarkus/deployment/KeycloakProcessor.java

@@ -52,9 +52,8 @@ import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
 import io.quarkus.agroal.spi.JdbcDataSourceBuildItem;
+import io.quarkus.datasource.deployment.spi.DevServicesDatasourceResultBuildItem;
 import io.quarkus.deployment.IsDevelopment;
-import io.quarkus.deployment.IsNormal;
-import io.quarkus.deployment.IsTest;
 import io.quarkus.deployment.annotations.Consume;
 import io.quarkus.deployment.builditem.CombinedIndexBuildItem;
 import io.quarkus.deployment.builditem.GeneratedResourceBuildItem;
@@ -84,6 +83,8 @@ import org.jboss.resteasy.plugins.server.servlet.ResteasyContextParameters;
 import org.jboss.resteasy.spi.ResteasyDeployment;
 import org.keycloak.Config;
 import org.keycloak.quarkus.runtime.configuration.PersistedConfigSource;
+import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMapper;
+import org.keycloak.quarkus.runtime.configuration.mappers.PropertyMappers;
 import org.keycloak.quarkus.runtime.integration.jaxrs.QuarkusKeycloakApplication;
 import org.keycloak.authentication.AuthenticatorSpi;
 import org.keycloak.authentication.authenticators.browser.DeployedScriptAuthenticatorFactory;
@@ -273,8 +274,32 @@ class KeycloakProcessor {
     }
 
     @BuildStep(onlyIf = IsIntegrationTest.class)
-    void prepareTestEnvironment(BuildProducer<StaticInitConfigSourceProviderBuildItem> configSources) {
+    void prepareTestEnvironment(BuildProducer<StaticInitConfigSourceProviderBuildItem> configSources, DevServicesDatasourceResultBuildItem dbConfig) {
         configSources.produce(new StaticInitConfigSourceProviderBuildItem("org.keycloak.quarkus.runtime.configuration.test.TestKeycloakConfigSourceProvider"));
+
+        // we do not enable dev services by default and the DevServicesDatasourceResultBuildItem might not be available when discovering build steps
+        // Quarkus seems to allow that when the DevServicesDatasourceResultBuildItem is not the only parameter to the build step
+        // this might be too sensitive and break if Quarkus changes the behavior
+        if (dbConfig != null && dbConfig.getDefaultDatasource() != null) {
+            Map<String, String> configProperties = dbConfig.getDefaultDatasource().getConfigProperties();
+
+            for (Entry<String, String> dbConfigProperty : configProperties.entrySet()) {
+                PropertyMapper mapper = PropertyMappers.getMapper(dbConfigProperty.getKey());
+
+                if (mapper == null) {
+                    continue;
+                }
+
+                String kcProperty = mapper.getFrom();
+
+                if (kcProperty.endsWith("db")) {
+                    // db kind set when running tests
+                    continue;
+                }
+
+                System.setProperty(kcProperty, dbConfigProperty.getValue());
+            }
+        }
     }
 
     /**

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/Picocli.java

@@ -395,8 +395,9 @@ public final class Picocli {
         while (iterator.hasNext()) {
             String arg = iterator.next();
 
-            if (arg.startsWith("--spi")) {
+            if (arg.startsWith("--spi") || arg.startsWith("-D")) {
                 // TODO: ignore properties for providers for now, need to fetch them from the providers, otherwise CLI will complain about invalid options
+                // also ignores system properties as they are set when starting the JVM
                 // change this once we are able to obtain properties from providers
                 iterator.remove();
 

quarkus/tests/integration/pom.xml

@@ -90,4 +90,27 @@
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>test-database</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <argLine>-Xmx1024m -XX:MaxMetaspaceSize=512m</argLine>
+                            <systemProperties>
+                                <kc.test.storage.database>true</kc.test.storage.database>
+                            </systemProperties>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
 </project>

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java

@@ -25,7 +25,9 @@ import static org.keycloak.quarkus.runtime.cli.command.Main.CONFIG_FILE_SHORT_NA
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import io.quarkus.dev.console.QuarkusConsole;
@@ -48,6 +50,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
 
     private static final String KEY_VALUE_SEPARATOR = "[= ]";
     private KeycloakDistribution dist;
+    private final Set<String> testSysProps = new HashSet<>();
 
     @Override
     public void beforeEach(ExtensionContext context) throws Exception {
@@ -59,13 +62,23 @@ public class CLITestExtension extends QuarkusMainTestExtension {
                 if (arg.contains(CONFIG_FILE_SHORT_NAME) || arg.contains(CONFIG_FILE_LONG_NAME)) {
                     Pattern kvSeparator = Pattern.compile(KEY_VALUE_SEPARATOR);
                     String[] cfKeyValue = kvSeparator.split(arg);
-                    System.setProperty(KeycloakPropertiesConfigSource.KEYCLOAK_CONFIG_FILE_PROP, cfKeyValue[1]);
+                    setProperty(KeycloakPropertiesConfigSource.KEYCLOAK_CONFIG_FILE_PROP, cfKeyValue[1]);
+                } else if (distConfig == null && arg.startsWith("-D")) {
+                    // allow setting system properties from JVM tests
+                    int keyValueSeparator = arg.indexOf('=');
+
+                    if (keyValueSeparator == -1) {
+                        continue;
+                    }
+
+                    String name = arg.substring(2, keyValueSeparator);
+                    String value = arg.substring(keyValueSeparator + 1);
+                    setProperty(name, value);
                 }
             }
         }
 
         if (distConfig != null) {
-
             if (launch != null) {
                 if (dist == null) {
                     dist = createDistribution(distConfig);
@@ -74,6 +87,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
             }
         } else {
             configureProfile(context);
+            configureDatabase(context);
             // WORKAROUND: this intercepts the output when actually starting the server.
             QuarkusConsole.installRedirects();
             super.beforeEach(context);
@@ -97,10 +111,12 @@ public class CLITestExtension extends QuarkusMainTestExtension {
     private void reset() {
         QuarkusConfigFactory.setConfig(null);
         //remove the config file property if set, and also the profile, to not have side effects in other tests.
-        System.getProperties().remove(KeycloakPropertiesConfigSource.KEYCLOAK_CONFIG_FILE_PROP);
         System.getProperties().remove(Environment.PROFILE);
         System.getProperties().remove("quarkus.profile");
         TestConfigArgsConfigSource.setCliArgs(new String[0]);
+        for (String property : testSysProps) {
+            System.getProperties().remove(property);
+        }
     }
 
     @Override
@@ -181,6 +197,28 @@ public class CLITestExtension extends QuarkusMainTestExtension {
         }
     }
 
+    private void configureDatabase(ExtensionContext context) {
+        WithDatabase database = context.getTestClass().orElse(Object.class).getDeclaredAnnotation(WithDatabase.class);
+
+        if (database != null) {
+            configureDevServices();
+            setProperty("kc.db", database.alias());
+            // databases like mssql are very strict about password policy
+            setProperty("kc.db.password", "Password1!");
+        }
+    }
+
+    private void configureDevServices() {
+        setProperty("quarkus.vault.devservices.enabled", Boolean.FALSE.toString());
+        setProperty("quarkus.datasource.devservices.enabled", Boolean.TRUE.toString());
+        setProperty("quarkus.devservices.enabled", Boolean.TRUE.toString());
+    }
+
+    private void setProperty(String name, String value) {
+        System.setProperty(name, value);
+        testSysProps.add(name);
+    }
+
     private List<String> getCliArgs(ExtensionContext context) {
         Launch annotation = context.getRequiredTestMethod().getAnnotation(Launch.class);
 

quarkus/tests/integration/src/main/java/org/keycloak/it/junit5/extension/WithDatabase.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.junit5.extension;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+
+/**
+ * {@link WithDatabase} is used to start a database container.
+ */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@EnabledIfSystemProperty(named = "kc.test.storage.database", matches = "true", disabledReason = "Docker takes too much time and stability depends on the environment. We should try running these tests in CI but isolated.")
+public @interface WithDatabase {
+
+    /**
+     * The database name as per database aliases.
+     *
+     * @return the database alias
+     */
+    String alias();
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/AbstractStartDabataseTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+public abstract class AbstractStartDabataseTest {
+
+    @Test
+    @Launch({ "start-dev" })
+    void testSuccessful(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertStartedDevMode();
+    }
+
+    @Test
+    @Launch({ "start-dev", "--db-username=wrong" })
+    void testWrongUsername(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertMessage("ERROR: Failed to obtain JDBC connection");
+        assertWrongUsername(cliResult);
+    }
+
+    protected abstract void assertWrongUsername(CLIResult cliResult);
+
+    @Test
+    @Launch({ "start-dev", "--db-password=wrong" })
+    void testWrongPassword(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertMessage("ERROR: Failed to obtain JDBC connection");
+        assertWrongPassword(cliResult);
+    }
+
+    protected abstract void assertWrongPassword(CLIResult cliResult);
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/MSSQLStartDatabaseTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+import org.keycloak.it.junit5.extension.CLITest;
+import org.keycloak.it.junit5.extension.WithDatabase;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+@CLITest
+@WithDatabase(alias = "mssql")
+public class MSSQLStartDatabaseTest extends AbstractStartDabataseTest {
+
+    /**
+     * It should be possible to enable XA but here we reproduce a managed environment where only nonXA transaction is supported
+     *
+     * @param result
+     */
+    @Override
+    @Test
+    @Launch({ "-Dkc.db.tx-type=enabled", "-Dkc.db.driver=com.microsoft.sqlserver.jdbc.SQLServerDriver", "start-dev" })
+    void testSuccessful(LaunchResult result) {
+        CLIResult cliResult = (CLIResult) result;
+        cliResult.assertStartedDevMode();
+    }
+
+    @Override
+    protected void assertWrongUsername(CLIResult cliResult) {
+        cliResult.assertMessage("ERROR: Login failed for user 'wrong'");
+    }
+
+    @Override
+    protected void assertWrongPassword(CLIResult cliResult) {
+        cliResult.assertMessage("Login failed for user");
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/MariaDBStartDatabaseTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+import org.keycloak.it.junit5.extension.CLITest;
+import org.keycloak.it.junit5.extension.WithDatabase;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+@CLITest
+@WithDatabase(alias = "mariadb")
+public class MariaDBStartDatabaseTest extends AbstractStartDabataseTest {
+
+    @Override
+    protected void assertWrongPassword(CLIResult cliResult) {
+        cliResult.assertMessage("Access denied for user");
+    }
+
+    @Override
+    protected void assertWrongUsername(CLIResult cliResult) {
+        cliResult.assertMessage("Access denied for user 'wrong'");
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/MySQLStartDatabaseTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+import org.keycloak.it.junit5.extension.CLITest;
+import org.keycloak.it.junit5.extension.WithDatabase;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+@CLITest
+@WithDatabase(alias = "mysql")
+public class MySQLStartDatabaseTest extends AbstractStartDabataseTest {
+
+    @Override
+    protected void assertWrongUsername(CLIResult cliResult) {
+        cliResult.assertMessage("ERROR: Access denied for user 'wrong'");
+    }
+
+    @Override
+    protected void assertWrongPassword(CLIResult cliResult) {
+        cliResult.assertMessage("ERROR: Access denied for user");
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/OracleStartDatabaseTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+import org.keycloak.it.junit5.extension.CLITest;
+import org.keycloak.it.junit5.extension.WithDatabase;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+@CLITest
+@WithDatabase(alias = "oracle")
+public class OracleStartDatabaseTest extends AbstractStartDabataseTest {
+
+    @Override
+    protected void assertWrongUsername(CLIResult cliResult) {
+        cliResult.assertMessage("ORA-01017: invalid username/password; logon denied");
+    }
+
+    @Override
+    protected void assertWrongPassword(CLIResult cliResult) {
+        cliResult.assertMessage("ORA-01017: invalid username/password; logon denied");
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/PostgreSQLStartDatabaseTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.it.storage.database;
+
+import org.junit.jupiter.api.Test;
+import org.keycloak.it.junit5.extension.CLIResult;
+import org.keycloak.it.junit5.extension.CLITest;
+import org.keycloak.it.junit5.extension.WithDatabase;
+
+import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
+
+@CLITest
+@WithDatabase(alias = "postgres")
+public class PostgreSQLStartDatabaseTest extends AbstractStartDabataseTest {
+
+    @Override
+    protected void assertWrongUsername(CLIResult cliResult) {
+        cliResult.assertMessage("ERROR: FATAL: password authentication failed for user \"wrong\"");
+    }
+
+    @Override
+    protected void assertWrongPassword(CLIResult cliResult) {
+        cliResult.assertMessage("ERROR: FATAL: password authentication failed for user");
+    }
+}

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/dist/CustomTransactionDistTest.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.keycloak.it.database.dist;
+package org.keycloak.it.storage.database.dist;
 
 import org.junit.jupiter.api.Test;
 import org.keycloak.it.junit5.extension.CLIResult;
@@ -25,7 +25,7 @@ import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 
 @DistributionTest
-public class CustomTransactionTest {
+public class CustomTransactionDistTest {
 
     @Test
     @Launch({ "-Dkc.db.tx-type=enabled", "-Dkc.db.driver=org.postgresql.xa.PGXADataSource", "build", "--db=postgres" })

quarkus/tests/integration/src/test/resources/application.properties

@@ -0,0 +1,19 @@
+#
+# Copyright 2021 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+quarkus.vault.devservices.enabled=false
+quarkus.datasource.devservices.enabled=false
+quarkus.devservices.enabled=true

quarkus/tests/integration/src/test/resources/container-license-acceptance.txt

@@ -0,0 +1 @@
+mcr.microsoft.com/mssql/server:2019-CU10-ubuntu-20.04