Make general cache options runtime (#28542)

Closes #27549

Signed-off-by: Martin Bartoš <mabartos@redhat.com>

docs/documentation/release_notes/topics/25_0_0.adoc

@@ -103,3 +103,10 @@ It utilizes the protocol defined in https://datatracker.ietf.org/doc/html/rfc542
 By default, the syslog handler is disabled, but when enabled, it sends all log events to a remote syslog server.
 
 For more information, see the https://www.keycloak.org/server/logging[Configuring logging] guide.
+
+= All `cache` options are runtime
+
+It is now possible to specify the `cache`, `cache-stack`, and `cache-config-file` options during runtime.
+This eliminates the need to execute the build phase and rebuild your image due to them.
+
+For more details, see the link:{upgradingguide_link}[{upgradingguide_name}].

docs/documentation/upgrading/topics/changes/changes-25_0_0.adoc

@@ -63,6 +63,12 @@ The module `org.keycloak:keycloak-model-legacy` module was deprecated in a previ
 
 The old behavior to preload offline sessions at startup is now removed after it has been deprecated in the previous release.
 
+= Specify `cache` options at runtime
+
+Options `cache`, `cache-stack`, and `cache-config-file` are no longer build options, and they can be specified only during runtime.
+This eliminates the need to execute the build phase and rebuild your image due to them.
+Be aware that they will not be recognized during the `build` phase, so you need to remove them.
+
 = kcadm Changes
 
 How kcadm parses and handles options and parameters has changed. Error messages from usage errors, the wrong option or parameter, may be slightly different than previous versions. Also usage errors will have an exit code of 2 instead of 1.

quarkus/config-api/src/main/java/org/keycloak/config/CachingOptions.java

@@ -36,7 +36,6 @@ public class CachingOptions {
                     + "By default in production mode, a 'ispn' cache is used to create a cluster between multiple server nodes. "
                     + "By default in development mode, a 'local' cache disables clustering and is intended for development and testing purposes.")
             .defaultValue(Mechanism.ispn)
-            .buildTime(true)
             .build();
 
     public enum Stack {
@@ -53,14 +52,12 @@ public class CachingOptions {
             .expectedValues(List.of())
             .description("Define the default stack to use for cluster communication and node discovery. This option only takes effect "
                     + "if 'cache' is set to 'ispn'. Default: udp. Built-in values include: " + Stream.of(Stack.values()).map(Stack::name).collect(Collectors.joining(", ")))
-            .buildTime(true)
             .build();
 
     public static final Option<File> CACHE_CONFIG_FILE = new OptionBuilder<>(CACHE_CONFIG_FILE_PROPERTY, File.class)
             .category(OptionCategory.CACHE)
             .description("Defines the file from which cache configuration should be loaded from. "
                     + "The configuration file is relative to the 'conf/' directory.")
-            .buildTime(true)
             .build();
 
     public static final Option<Boolean> CACHE_EMBEDDED_MTLS_ENABLED = new OptionBuilder<>(CACHE_EMBEDDED_MTLS_ENABLED_PROPERTY, Boolean.class)

quarkus/deployment/src/main/java/org/keycloak/quarkus/deployment/CacheBuildSteps.java

@@ -17,20 +17,8 @@
 
 package org.keycloak.quarkus.deployment;
 
-import static org.keycloak.quarkus.runtime.configuration.Configuration.getConfigValue;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.stream.Collectors;
 import jakarta.enterprise.context.ApplicationScoped;
-import org.infinispan.commons.util.FileLookupFactory;
-import org.keycloak.config.MetricsOptions;
 import org.keycloak.quarkus.runtime.KeycloakRecorder;
-import org.keycloak.quarkus.runtime.configuration.Configuration;
-import org.keycloak.quarkus.runtime.configuration.MicroProfileConfigProvider;
 import org.keycloak.quarkus.runtime.storage.legacy.infinispan.CacheManagerFactory;
 
 import io.quarkus.arc.deployment.SyntheticBeanBuildItem;
@@ -47,41 +35,10 @@ public class CacheBuildSteps {
     @Record(ExecutionTime.STATIC_INIT)
     @BuildStep
     void configureInfinispan(KeycloakRecorder recorder, BuildProducer<SyntheticBeanBuildItem> syntheticBeanBuildItems, ShutdownContextBuildItem shutdownContext) {
-        String configFile = getConfigValue("kc.spi-connections-infinispan-quarkus-config-file").getValue();
-
-        if (configFile != null) {
-            Path configPath = Paths.get(configFile);
-            String path;
-
-            if (configPath.toFile().exists()) {
-                path = configPath.toFile().getAbsolutePath();
-            } else {
-                path = configPath.getFileName().toString();
-            }
-
-            InputStream url = FileLookupFactory.newInstance().lookupFile(path, KeycloakProcessor.class.getClassLoader());
-
-            if (url == null) {
-                throw new IllegalArgumentException("Could not load cluster configuration file at [" + configPath + "]");
-            }
-
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(url))) {
-                String config = reader.lines().collect(Collectors.joining("\n"));
-
         syntheticBeanBuildItems.produce(SyntheticBeanBuildItem.configure(CacheManagerFactory.class)
                 .scope(ApplicationScoped.class)
                 .unremovable()
                 .setRuntimeInit()
-                        .runtimeValue(recorder.createCacheInitializer(config, isMetricsEnabled(), shutdownContext)).done());
+                .runtimeValue(recorder.createCacheInitializer(shutdownContext)).done());
-            } catch (Exception cause) {
-                throw new RuntimeException("Failed to read clustering configuration from [" + url + "]", cause);
-            }
-        } else {
-            throw new IllegalArgumentException("Option 'configFile' needs to be specified");
-        }
-    }
-
-    private boolean isMetricsEnabled() {
-        return Configuration.getOptionalBooleanValue(MicroProfileConfigProvider.NS_KEYCLOAK_PREFIX.concat(MetricsOptions.METRICS_ENABLED.getKey())).orElse(false);
     }
 }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/KeycloakRecorder.java

@@ -17,12 +17,18 @@
 
 package org.keycloak.quarkus.runtime;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.lang.annotation.Annotation;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import io.agroal.api.AgroalDataSource;
@@ -35,6 +41,7 @@ import io.vertx.ext.web.RoutingContext;
 import liquibase.Scope;
 
 import org.hibernate.cfg.AvailableSettings;
+import org.infinispan.commons.util.FileLookupFactory;
 import org.infinispan.manager.DefaultCacheManager;
 
 import org.keycloak.Config;
@@ -42,6 +49,7 @@ import org.keycloak.common.Profile;
 import org.keycloak.common.crypto.CryptoIntegration;
 import org.keycloak.common.crypto.CryptoProvider;
 import org.keycloak.common.crypto.FipsMode;
+import org.keycloak.config.MetricsOptions;
 import org.keycloak.config.TruststoreOptions;
 import org.keycloak.quarkus.runtime.configuration.Configuration;
 import org.keycloak.quarkus.runtime.configuration.MicroProfileConfigProvider;
@@ -61,6 +69,8 @@ import io.quarkus.runtime.annotations.Recorder;
 import liquibase.servicelocator.ServiceLocator;
 import org.keycloak.userprofile.DeclarativeUserProfileProviderFactory;
 
+import static org.keycloak.quarkus.runtime.configuration.Configuration.getKcConfigValue;
+
 @Recorder
 public class KeycloakRecorder {
 
@@ -111,19 +121,17 @@ public class KeycloakRecorder {
         QuarkusKeycloakSessionFactory.setInstance(new QuarkusKeycloakSessionFactory(factories, defaultProviders, preConfiguredProviders, themes, reaugmented));
     }
 
-    public RuntimeValue<CacheManagerFactory> createCacheInitializer(String config, boolean metricsEnabled, ShutdownContext shutdownContext) {
+    public RuntimeValue<CacheManagerFactory> createCacheInitializer(ShutdownContext shutdownContext) {
         try {
-            CacheManagerFactory cacheManagerFactory = new CacheManagerFactory(config, metricsEnabled);
+            boolean isMetricsEnabled = Configuration.isTrue(MetricsOptions.METRICS_ENABLED);
+            CacheManagerFactory cacheManagerFactory = new CacheManagerFactory(getInfinispanConfigFile(), isMetricsEnabled);
 
-            shutdownContext.addShutdownTask(new Runnable() {
+            shutdownContext.addShutdownTask(() -> {
-                @Override
-                public void run() {
                 DefaultCacheManager cacheManager = cacheManagerFactory.getOrCreate();
 
                 if (cacheManager != null) {
                     cacheManager.stop();
                 }
-                }
             });
 
             return new RuntimeValue<>(cacheManagerFactory);
@@ -132,6 +140,35 @@ public class KeycloakRecorder {
         }
     }
 
+    private String getInfinispanConfigFile() {
+        String configFile = getKcConfigValue("spi-connections-infinispan-quarkus-config-file").getValue();
+
+        if (configFile != null) {
+            Path configPath = Paths.get(configFile);
+            String path;
+
+            if (configPath.toFile().exists()) {
+                path = configPath.toFile().getAbsolutePath();
+            } else {
+                path = configPath.getFileName().toString();
+            }
+
+            InputStream url = FileLookupFactory.newInstance().lookupFile(path, KeycloakRecorder.class.getClassLoader());
+
+            if (url == null) {
+                throw new IllegalArgumentException("Could not load cluster configuration file at [" + configPath + "]");
+            }
+
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(url))) {
+                return reader.lines().collect(Collectors.joining("\n"));
+            } catch (Exception cause) {
+                throw new RuntimeException("Failed to read clustering configuration from [" + url + "]", cause);
+            }
+        } else {
+            throw new IllegalArgumentException("Option 'configFile' needs to be specified");
+        }
+    }
+
     public void setDefaultUserProfileConfiguration(UPConfig configuration) {
         DeclarativeUserProfileProviderFactory.setDefaultConfig(configuration);
     }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/BuildCommandDistTest.java

@@ -90,10 +90,10 @@ class BuildCommandDistTest {
     @RawDistOnly(reason = "Containers are immutable")
     void testDoNotRecordRuntimeOptionsDuringBuild(KeycloakDistribution distribution) {
         distribution.setProperty("proxy", "edge");
-        distribution.run("build", "--cache=local");
+        distribution.run("build");
         distribution.removeProperty("proxy");
 
-        CLIResult result = distribution.run("start", "--hostname=mykeycloak", OPTIMIZED_BUILD_OPTION_LONG);
+        CLIResult result = distribution.run("start", "--hostname=mykeycloak", "--cache=local", OPTIMIZED_BUILD_OPTION_LONG);
         result.assertError("Key material not provided to setup HTTPS");
     }
 

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ClusterConfigDistTest.java

@@ -50,7 +50,7 @@ public class ClusterConfigDistTest {
     }
 
     @Test
-    @Launch({ "build", "--cache-config-file=invalid" })
+    @Launch({ "start-dev", "--cache-config-file=invalid" })
     void failInvalidClusterConfig(LaunchResult result) {
         assertTrue(result.getErrorOutput().contains("ERROR: Could not load cluster configuration file"));
     }

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/StartCommandDistTest.java

@@ -103,10 +103,10 @@ public class StartCommandDistTest {
     }
 
     @Test
-    @Launch({ "start", "--optimized", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
+    @Launch({ "start", "--optimized", "--http-enabled=true", "--hostname-strict=false", "--db=postgres" })
     void testStartUsingOptimizedDoesNotAllowBuildOptions(LaunchResult result) {
         CLIResult cliResult = (CLIResult) result;
-        cliResult.assertError("Build time option: '--cache' not usable with pre-built image and --optimized");
+        cliResult.assertError("Build time option: '--db' not usable with pre-built image and --optimized");
     }
 
     @Test
@@ -120,11 +120,10 @@ public class StartCommandDistTest {
     @Test
     @RawDistOnly(reason = "Containers are immutable")
     void testWarningWhenOverridingBuildOptionsDuringStart(KeycloakDistribution dist) {
-        CLIResult cliResult = dist.run("build", "--db=postgres", "--cache=local", "--features=preview");
+        CLIResult cliResult = dist.run("build", "--db=postgres", "--features=preview");
         cliResult.assertBuild();
         cliResult = dist.run("start", "--hostname=localhost", "--http-enabled=true");
         cliResult.assertMessage("The previous optimized build will be overridden with the following build options:");
-        cliResult.assertMessage("- cache=local > cache=ispn"); // back to the default value
         cliResult.assertMessage("- db=postgres > db=dev-file"); // back to the default value
         cliResult.assertMessage("- features=preview > features=<unset>"); // no default value, the <unset> is shown
         cliResult.assertMessage("To avoid that, run the 'build' command again and then start the optimized server instance using the '--optimized' flag.");
@@ -137,19 +136,17 @@ public class StartCommandDistTest {
         dist.run("build", "--db=postgres");
         cliResult = dist.run("start", "--hostname=localhost", "--http-enabled=true");
         cliResult.assertMessage("- db=postgres > db=dev-file");
-        cliResult.assertNoMessage("- cache=local > cache=ispn");
         cliResult.assertNoMessage("- features=preview > features=<unset>");
         cliResult.assertStarted();
         dist.run("build", "--db=postgres");
         cliResult = dist.run("start", "--db=dev-mem", "--hostname=localhost", "--http-enabled=true");
         cliResult.assertMessage("- db=postgres > db=dev-mem"); // option overridden during the start
         cliResult.assertStarted();
-        dist.run("build", "--db=dev-mem", "--cache=local");
+        dist.run("build", "--db=dev-mem");
         cliResult = dist.run("start", "--db=dev-mem", "--hostname=localhost", "--http-enabled=true");
         cliResult.assertNoMessage("- db=postgres > db=postgres"); // option did not change not need to show
-        cliResult.assertMessage("- cache=local > cache=ispn");
         cliResult.assertStarted();
-        dist.run("build", "--db=dev-mem", "--cache=local");
+        dist.run("build", "--db=dev-mem");
         cliResult = dist.run("start", "--db=dev-mem", "--cache=local", "--hostname=localhost", "--http-enabled=true");
         cliResult.assertNoMessage("The previous optimized build will be overridden with the following build options:"); // no message, same values provided during auto-build
     }

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testBuildHelp.approved.txt

@@ -17,21 +17,6 @@ Options:
 --help-all           This same help message but with additional options.
 -v, --verbose        Print out error details when running this command.
 
-Cache:
-
---cache <type>       Defines the cache mechanism for high-availability. By default in production
-                       mode, a 'ispn' cache is used to create a cluster between multiple server
-                       nodes. By default in development mode, a 'local' cache disables clustering
-                       and is intended for development and testing purposes. Possible values are:
-                       ispn, local. Default: ispn.
---cache-config-file <file>
-                     Defines the file from which cache configuration should be loaded from. The
-                       configuration file is relative to the 'conf/' directory.
---cache-stack <stack>
-                     Define the default stack to use for cluster communication and node discovery.
-                       This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
-                       Built-in values include: tcp, udp, kubernetes, ec2, azure, google
-
 Database:
 
 --db <vendor>        The database vendor. Possible values are: dev-file, dev-mem, mariadb, mssql,

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelp.approved.txt

@@ -18,6 +18,14 @@ Options:
 
 Cache:
 
+--cache <type>       Defines the cache mechanism for high-availability. By default in production
+                       mode, a 'ispn' cache is used to create a cluster between multiple server
+                       nodes. By default in development mode, a 'local' cache disables clustering
+                       and is intended for development and testing purposes. Possible values are:
+                       ispn, local. Default: ispn.
+--cache-config-file <file>
+                     Defines the file from which cache configuration should be loaded from. The
+                       configuration file is relative to the 'conf/' directory.
 --cache-embedded-mtls-enabled <true|false>
                      Encrypts the network communication between Keycloak servers. Default: false.
 --cache-embedded-mtls-key-store-file <file>
@@ -54,6 +62,10 @@ Cache:
                        specified via XML file (see 'cache-config-file' option.). If the option is
                        specified, 'cache-remote-host' and 'cache-remote-password' are required as
                        well and the related configuration in XML file should not be present.
+--cache-stack <stack>
+                     Define the default stack to use for cluster communication and node discovery.
+                       This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
+                       Built-in values include: tcp, udp, kubernetes, ec2, azure, google
 
 Config:
 

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.approved.txt

@@ -18,6 +18,14 @@ Options:
 
 Cache:
 
+--cache <type>       Defines the cache mechanism for high-availability. By default in production
+                       mode, a 'ispn' cache is used to create a cluster between multiple server
+                       nodes. By default in development mode, a 'local' cache disables clustering
+                       and is intended for development and testing purposes. Possible values are:
+                       ispn, local. Default: ispn.
+--cache-config-file <file>
+                     Defines the file from which cache configuration should be loaded from. The
+                       configuration file is relative to the 'conf/' directory.
 --cache-embedded-mtls-enabled <true|false>
                      Encrypts the network communication between Keycloak servers. Default: false.
 --cache-embedded-mtls-key-store-file <file>
@@ -57,6 +65,10 @@ Cache:
                        specified via XML file (see 'cache-config-file' option.). If the option is
                        specified, 'cache-remote-host' and 'cache-remote-password' are required as
                        well and the related configuration in XML file should not be present.
+--cache-stack <stack>
+                     Define the default stack to use for cluster communication and node discovery.
+                       This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
+                       Built-in values include: tcp, udp, kubernetes, ec2, azure, google
 
 Config:
 

quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/ServerOptions.java

@@ -39,11 +39,15 @@ final class ServerOptions extends ArrayList<String> {
             .or("-h"::equals)
             .or(ShowConfig.NAME::equals);
 
+    private boolean isBuildPhase = false;
+
     ServerOptions(Storage storageConfig, WithDatabase withDatabase, List<String> rawOptions) {
         if (rawOptions.isEmpty()) {
             return;
         }
 
+        this.isBuildPhase = rawOptions.contains("build");
+
         for (Map.Entry<String, Predicate<String>> entry : getDefaultOptions(storageConfig, withDatabase).entrySet()) {
             if (contains(entry.getKey())) {
                 continue;
@@ -60,7 +64,9 @@ final class ServerOptions extends ArrayList<String> {
     private Map<String, Predicate<String>> getDefaultOptions(Storage storageConfig, WithDatabase withDatabase) {
         Map<String, Predicate<String>> defaultOptions = new HashMap<>();
 
+        if (!isBuildPhase) {
             defaultOptions.put("--cache=local", ignoreCacheLocal(storageConfig));
+        }
 
         return defaultOptions;
     }

testsuite/integration-arquillian/servers/auth-server/quarkus/ant/configure.xml

@@ -6,12 +6,10 @@
         <exec osfamily="unix" dir="${auth.server.home}/bin" executable="./kc.sh" failonerror="true">
             <arg value="build"/>
             <arg value="--http-relative-path=/auth"/>
-            <arg value="--cache=local"/>
         </exec>
         <exec osfamily="windows" executable="${auth.server.home}/bin/kc.bat" failonerror="true">
             <arg value="build"/>
             <arg value="--http-relative-path=/auth"/>
-            <arg value="--cache=local"/>
         </exec>
     </target>
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/AbstractQuarkusDeployableContainer.java

@@ -185,10 +185,12 @@ public abstract class AbstractQuarkusDeployableContainer implements DeployableCo
         final String cacheMode = System.getProperty("auth.server.quarkus.cluster.config", "local");
 
         if ("local".equals(cacheMode)) {
+            commands.add("--cache=local");
             // Save ~2s for each Quarkus startup, when we know ISPN cluster is empty. See https://github.com/keycloak/keycloak/issues/21033
             commands.add("-Djgroups.join_timeout=10");
         } else {
             commands.add("--cache=ispn");
+            commands.add("--cache-config-file=cluster-" + cacheMode + ".xml");
         }
 
         log.debugf("FIPS Mode: %s", configuration.getFipsMode());
@@ -197,12 +199,6 @@ public abstract class AbstractQuarkusDeployableContainer implements DeployableCo
         if (restart.get() || "ha".equals(cacheMode) || shouldSetUpDb.get() || configuration.getFipsMode() != FipsMode.DISABLED) {
             prepareCommandsForRebuilding(commands);
 
-            if ("local".equals(cacheMode)) {
-                commands.add("--cache=local");
-            } else {
-                commands.add("--cache-config-file=cluster-" + cacheMode + ".xml");
-            }
-
             if (configuration.getFipsMode() != FipsMode.DISABLED) {
                 addFipsOptions(commands);
             }