libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases 1 Packages Activity Actions

Make general cache options runtime (#28542)

Browse source 
Closes #27549

Signed-off-by: Martin Bartoš <mabartos@redhat.com>
This commit is contained in:
Martin Bartoš 2024-04-12 11:56:11 +02:00 committed by GitHub
parent 6d74e6b289
commit a3669a6562
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 106 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
docs/documentation/release_notes/topics/25_0_0.adoc
View file

@ -102,4 +102,11 @@ For more details, see https://www.keycloak.org/server/management-interface[Confi
It utilizes the protocol defined in https://datatracker.ietf.org/doc/html/rfc5424[RFC 5424].
By default, the syslog handler is disabled, but when enabled, it sends all log events to a remote syslog server.
For more information, see the https://www.keycloak.org/server/logging[Configuring logging] guide.
For more information, see the https://www.keycloak.org/server/logging[Configuring logging] guide.
= All `cache` options are runtime
It is now possible to specify the `cache`, `cache-stack`, and `cache-config-file` options during runtime.
This eliminates the need to execute the build phase and rebuild your image due to them.
For more details, see the link:{upgradingguide_link}[{upgradingguide_name}].

6
docs/documentation/upgrading/topics/changes/changes-25_0_0.adoc
View file

@ -63,6 +63,12 @@ The module `org.keycloak:keycloak-model-legacy` module was deprecated in a previ
The old behavior to preload offline sessions at startup is now removed after it has been deprecated in the previous release.
= Specify `cache` options at runtime
Options `cache`, `cache-stack`, and `cache-config-file` are no longer build options, and they can be specified only during runtime.
This eliminates the need to execute the build phase and rebuild your image due to them.
Be aware that they will not be recognized during the `build` phase, so you need to remove them.
= kcadm Changes
How kcadm parses and handles options and parameters has changed. Error messages from usage errors, the wrong option or parameter, may be slightly different than previous versions. Also usage errors will have an exit code of 2 instead of 1.

3
quarkus/config-api/src/main/java/org/keycloak/config/CachingOptions.java
View file

@ -36,7 +36,6 @@ public class CachingOptions {
+ "By default in production mode, a 'ispn' cache is used to create a cluster between multiple server nodes. "
+ "By default in development mode, a 'local' cache disables clustering and is intended for development and testing purposes.")
.defaultValue(Mechanism.ispn)
.buildTime(true)
.build();
public enum Stack {
@ -53,14 +52,12 @@ public class CachingOptions {
.expectedValues(List.of())
.description("Define the default stack to use for cluster communication and node discovery. This option only takes effect "
+ "if 'cache' is set to 'ispn'. Default: udp. Built-in values include: " + Stream.of(Stack.values()).map(Stack::name).collect(Collectors.joining(", ")))
.buildTime(true)
.build();
public static final Option<File> CACHE_CONFIG_FILE = new OptionBuilder<>(CACHE_CONFIG_FILE_PROPERTY, File.class)
.category(OptionCategory.CACHE)
.description("Defines the file from which cache configuration should be loaded from. "
+ "The configuration file is relative to the 'conf/' directory.")
.buildTime(true)
.build();
public static final Option<Boolean> CACHE_EMBEDDED_MTLS_ENABLED = new OptionBuilder<>(CACHE_EMBEDDED_MTLS_ENABLED_PROPERTY, Boolean.class)

53
quarkus/deployment/src/main/java/org/keycloak/quarkus/deployment/CacheBuildSteps.java
View file

@ -17,20 +17,8 @@
package org.keycloak.quarkus.deployment;
import static org.keycloak.quarkus.runtime.configuration.Configuration.getConfigValue;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Collectors;
import jakarta.enterprise.context.ApplicationScoped;
import org.infinispan.commons.util.FileLookupFactory;
import org.keycloak.config.MetricsOptions;
import org.keycloak.quarkus.runtime.KeycloakRecorder;
import org.keycloak.quarkus.runtime.configuration.Configuration;
import org.keycloak.quarkus.runtime.configuration.MicroProfileConfigProvider;
import org.keycloak.quarkus.runtime.storage.legacy.infinispan.CacheManagerFactory;
import io.quarkus.arc.deployment.SyntheticBeanBuildItem;
@ -47,41 +35,10 @@ public class CacheBuildSteps {
@Record(ExecutionTime.STATIC_INIT)
@BuildStep
void configureInfinispan(KeycloakRecorder recorder, BuildProducer<SyntheticBeanBuildItem> syntheticBeanBuildItems, ShutdownContextBuildItem shutdownContext) {
String configFile = getConfigValue("kc.spi-connections-infinispan-quarkus-config-file").getValue();
if (configFile != null) {
Path configPath = Paths.get(configFile);
String path;
if (configPath.toFile().exists()) {
path = configPath.toFile().getAbsolutePath();
} else {
path = configPath.getFileName().toString();
}
InputStream url = FileLookupFactory.newInstance().lookupFile(path, KeycloakProcessor.class.getClassLoader());
if (url == null) {
throw new IllegalArgumentException("Could not load cluster configuration file at [" + configPath + "]");
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(url))) {
String config = reader.lines().collect(Collectors.joining("\n"));
syntheticBeanBuildItems.produce(SyntheticBeanBuildItem.configure(CacheManagerFactory.class)
.scope(ApplicationScoped.class)
.unremovable()
.setRuntimeInit()
.runtimeValue(recorder.createCacheInitializer(config, isMetricsEnabled(), shutdownContext)).done());
} catch (Exception cause) {
throw new RuntimeException("Failed to read clustering configuration from [" + url + "]", cause);
}
} else {
throw new IllegalArgumentException("Option 'configFile' needs to be specified");
}
}
private boolean isMetricsEnabled() {
return Configuration.getOptionalBooleanValue(MicroProfileConfigProvider.NS_KEYCLOAK_PREFIX.concat(MetricsOptions.METRICS_ENABLED.getKey())).orElse(false);
syntheticBeanBuildItems.produce(SyntheticBeanBuildItem.configure(CacheManagerFactory.class)
.scope(ApplicationScoped.class)
.unremovable()
.setRuntimeInit()
.runtimeValue(recorder.createCacheInitializer(shutdownContext)).done());
}
}

55
quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/KeycloakRecorder.java
View file

@ -17,12 +17,18 @@
package org.keycloak.quarkus.runtime;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.annotation.Annotation;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import io.agroal.api.AgroalDataSource;
@ -35,6 +41,7 @@ import io.vertx.ext.web.RoutingContext;
import liquibase.Scope;
import org.hibernate.cfg.AvailableSettings;
import org.infinispan.commons.util.FileLookupFactory;
import org.infinispan.manager.DefaultCacheManager;
import org.keycloak.Config;
@ -42,6 +49,7 @@ import org.keycloak.common.Profile;
import org.keycloak.common.crypto.CryptoIntegration;
import org.keycloak.common.crypto.CryptoProvider;
import org.keycloak.common.crypto.FipsMode;
import org.keycloak.config.MetricsOptions;
import org.keycloak.config.TruststoreOptions;
import org.keycloak.quarkus.runtime.configuration.Configuration;
import org.keycloak.quarkus.runtime.configuration.MicroProfileConfigProvider;
@ -61,6 +69,8 @@ import io.quarkus.runtime.annotations.Recorder;
import liquibase.servicelocator.ServiceLocator;
import org.keycloak.userprofile.DeclarativeUserProfileProviderFactory;
import static org.keycloak.quarkus.runtime.configuration.Configuration.getKcConfigValue;
@Recorder
public class KeycloakRecorder {
@ -111,18 +121,16 @@ public class KeycloakRecorder {
QuarkusKeycloakSessionFactory.setInstance(new QuarkusKeycloakSessionFactory(factories, defaultProviders, preConfiguredProviders, themes, reaugmented));
}
public RuntimeValue<CacheManagerFactory> createCacheInitializer(String config, boolean metricsEnabled, ShutdownContext shutdownContext) {
public RuntimeValue<CacheManagerFactory> createCacheInitializer(ShutdownContext shutdownContext) {
try {
CacheManagerFactory cacheManagerFactory = new CacheManagerFactory(config, metricsEnabled);
boolean isMetricsEnabled = Configuration.isTrue(MetricsOptions.METRICS_ENABLED);
CacheManagerFactory cacheManagerFactory = new CacheManagerFactory(getInfinispanConfigFile(), isMetricsEnabled);
shutdownContext.addShutdownTask(new Runnable() {
@Override
public void run() {
DefaultCacheManager cacheManager = cacheManagerFactory.getOrCreate();
shutdownContext.addShutdownTask(() -> {
DefaultCacheManager cacheManager = cacheManagerFactory.getOrCreate();
if (cacheManager != null) {
cacheManager.stop();
}
if (cacheManager != null) {
cacheManager.stop();
}
});
@ -132,6 +140,35 @@ public class KeycloakRecorder {
}
}
private String getInfinispanConfigFile() {
String configFile = getKcConfigValue("spi-connections-infinispan-quarkus-config-file").getValue();
if (configFile != null) {
Path configPath = Paths.get(configFile);
String path;
if (configPath.toFile().exists()) {
path = configPath.toFile().getAbsolutePath();
} else {
path = configPath.getFileName().toString();
}
InputStream url = FileLookupFactory.newInstance().lookupFile(path, KeycloakRecorder.class.getClassLoader());
if (url == null) {
throw new IllegalArgumentException("Could not load cluster configuration file at [" + configPath + "]");
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(url))) {
return reader.lines().collect(Collectors.joining("\n"));
} catch (Exception cause) {
throw new RuntimeException("Failed to read clustering configuration from [" + url + "]", cause);
}
} else {
throw new IllegalArgumentException("Option 'configFile' needs to be specified");
}
}
public void setDefaultUserProfileConfiguration(UPConfig configuration) {
DeclarativeUserProfileProviderFactory.setDefaultConfig(configuration);
}

4
quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/BuildCommandDistTest.java vendored
View file

@ -90,10 +90,10 @@ class BuildCommandDistTest {
@RawDistOnly(reason = "Containers are immutable")
void testDoNotRecordRuntimeOptionsDuringBuild(KeycloakDistribution distribution) {
distribution.setProperty("proxy", "edge");
distribution.run("build", "--cache=local");
distribution.run("build");
distribution.removeProperty("proxy");
CLIResult result = distribution.run("start", "--hostname=mykeycloak", OPTIMIZED_BUILD_OPTION_LONG);
CLIResult result = distribution.run("start", "--hostname=mykeycloak", "--cache=local", OPTIMIZED_BUILD_OPTION_LONG);
result.assertError("Key material not provided to setup HTTPS");
}

2
quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/ClusterConfigDistTest.java vendored
View file

@ -50,7 +50,7 @@ public class ClusterConfigDistTest {
}
@Test
@Launch({ "build", "--cache-config-file=invalid" })
@Launch({ "start-dev", "--cache-config-file=invalid" })
void failInvalidClusterConfig(LaunchResult result) {
assertTrue(result.getErrorOutput().contains("ERROR: Could not load cluster configuration file"));
}

13
quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/StartCommandDistTest.java vendored
View file

@ -103,10 +103,10 @@ public class StartCommandDistTest {
}
@Test
@Launch({ "start", "--optimized", "--http-enabled=true", "--hostname-strict=false", "--cache=local" })
@Launch({ "start", "--optimized", "--http-enabled=true", "--hostname-strict=false", "--db=postgres" })
void testStartUsingOptimizedDoesNotAllowBuildOptions(LaunchResult result) {
CLIResult cliResult = (CLIResult) result;
cliResult.assertError("Build time option: '--cache' not usable with pre-built image and --optimized");
cliResult.assertError("Build time option: '--db' not usable with pre-built image and --optimized");
}
@Test
@ -120,11 +120,10 @@ public class StartCommandDistTest {
@Test
@RawDistOnly(reason = "Containers are immutable")
void testWarningWhenOverridingBuildOptionsDuringStart(KeycloakDistribution dist) {
CLIResult cliResult = dist.run("build", "--db=postgres", "--cache=local", "--features=preview");
CLIResult cliResult = dist.run("build", "--db=postgres", "--features=preview");
cliResult.assertBuild();
cliResult = dist.run("start", "--hostname=localhost", "--http-enabled=true");
cliResult.assertMessage("The previous optimized build will be overridden with the following build options:");
cliResult.assertMessage("- cache=local > cache=ispn"); // back to the default value
cliResult.assertMessage("- db=postgres > db=dev-file"); // back to the default value
cliResult.assertMessage("- features=preview > features=<unset>"); // no default value, the <unset> is shown
cliResult.assertMessage("To avoid that, run the 'build' command again and then start the optimized server instance using the '--optimized' flag.");
@ -137,19 +136,17 @@ public class StartCommandDistTest {
dist.run("build", "--db=postgres");
cliResult = dist.run("start", "--hostname=localhost", "--http-enabled=true");
cliResult.assertMessage("- db=postgres > db=dev-file");
cliResult.assertNoMessage("- cache=local > cache=ispn");
cliResult.assertNoMessage("- features=preview > features=<unset>");
cliResult.assertStarted();
dist.run("build", "--db=postgres");
cliResult = dist.run("start", "--db=dev-mem", "--hostname=localhost", "--http-enabled=true");
cliResult.assertMessage("- db=postgres > db=dev-mem"); // option overridden during the start
cliResult.assertStarted();
dist.run("build", "--db=dev-mem", "--cache=local");
dist.run("build", "--db=dev-mem");
cliResult = dist.run("start", "--db=dev-mem", "--hostname=localhost", "--http-enabled=true");
cliResult.assertNoMessage("- db=postgres > db=postgres"); // option did not change not need to show
cliResult.assertMessage("- cache=local > cache=ispn");
cliResult.assertStarted();
dist.run("build", "--db=dev-mem", "--cache=local");
dist.run("build", "--db=dev-mem");
cliResult = dist.run("start", "--db=dev-mem", "--cache=local", "--hostname=localhost", "--http-enabled=true");
cliResult.assertNoMessage("The previous optimized build will be overridden with the following build options:"); // no message, same values provided during auto-build
}

15
quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testBuildHelp.approved.txt vendored
View file

@ -17,21 +17,6 @@ Options:
--help-all This same help message but with additional options.
-v, --verbose Print out error details when running this command.
Cache:
--cache <type> Defines the cache mechanism for high-availability. By default in production
mode, a 'ispn' cache is used to create a cluster between multiple server
nodes. By default in development mode, a 'local' cache disables clustering
and is intended for development and testing purposes. Possible values are:
ispn, local. Default: ispn.
--cache-config-file <file>
Defines the file from which cache configuration should be loaded from. The
configuration file is relative to the 'conf/' directory.
--cache-stack <stack>
Define the default stack to use for cluster communication and node discovery.
This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
Built-in values include: tcp, udp, kubernetes, ec2, azure, google
Database:
--db <vendor> The database vendor. Possible values are: dev-file, dev-mem, mariadb, mssql,

12
quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelp.approved.txt vendored
View file

@ -18,6 +18,14 @@ Options:
Cache:
--cache <type> Defines the cache mechanism for high-availability. By default in production
mode, a 'ispn' cache is used to create a cluster between multiple server
nodes. By default in development mode, a 'local' cache disables clustering
and is intended for development and testing purposes. Possible values are:
ispn, local. Default: ispn.
--cache-config-file <file>
Defines the file from which cache configuration should be loaded from. The
configuration file is relative to the 'conf/' directory.
--cache-embedded-mtls-enabled <true|false>
Encrypts the network communication between Keycloak servers. Default: false.
--cache-embedded-mtls-key-store-file <file>
@ -54,6 +62,10 @@ Cache:
specified via XML file (see 'cache-config-file' option.). If the option is
specified, 'cache-remote-host' and 'cache-remote-password' are required as
well and the related configuration in XML file should not be present.
--cache-stack <stack>
Define the default stack to use for cluster communication and node discovery.
This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
Built-in values include: tcp, udp, kubernetes, ec2, azure, google
Config:

12
quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartOptimizedHelpAll.approved.txt vendored
View file

@ -18,6 +18,14 @@ Options:
Cache:
--cache <type> Defines the cache mechanism for high-availability. By default in production
mode, a 'ispn' cache is used to create a cluster between multiple server
nodes. By default in development mode, a 'local' cache disables clustering
and is intended for development and testing purposes. Possible values are:
ispn, local. Default: ispn.
--cache-config-file <file>
Defines the file from which cache configuration should be loaded from. The
configuration file is relative to the 'conf/' directory.
--cache-embedded-mtls-enabled <true|false>
Encrypts the network communication between Keycloak servers. Default: false.
--cache-embedded-mtls-key-store-file <file>
@ -57,6 +65,10 @@ Cache:
specified via XML file (see 'cache-config-file' option.). If the option is
specified, 'cache-remote-host' and 'cache-remote-password' are required as
well and the related configuration in XML file should not be present.
--cache-stack <stack>
Define the default stack to use for cluster communication and node discovery.
This option only takes effect if 'cache' is set to 'ispn'. Default: udp.
Built-in values include: tcp, udp, kubernetes, ec2, azure, google
Config:

8
quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/ServerOptions.java
View file

@ -39,11 +39,15 @@ final class ServerOptions extends ArrayList<String> {
.or("-h"::equals)
.or(ShowConfig.NAME::equals);
private boolean isBuildPhase = false;
ServerOptions(Storage storageConfig, WithDatabase withDatabase, List<String> rawOptions) {
if (rawOptions.isEmpty()) {
return;
}
this.isBuildPhase = rawOptions.contains("build");
for (Map.Entry<String, Predicate<String>> entry : getDefaultOptions(storageConfig, withDatabase).entrySet()) {
if (contains(entry.getKey())) {
continue;
@ -60,7 +64,9 @@ final class ServerOptions extends ArrayList<String> {
private Map<String, Predicate<String>> getDefaultOptions(Storage storageConfig, WithDatabase withDatabase) {
Map<String, Predicate<String>> defaultOptions = new HashMap<>();
defaultOptions.put("--cache=local", ignoreCacheLocal(storageConfig));
if (!isBuildPhase) {
defaultOptions.put("--cache=local", ignoreCacheLocal(storageConfig));
}
return defaultOptions;
}

2
testsuite/integration-arquillian/servers/auth-server/quarkus/ant/configure.xml
View file

@ -6,12 +6,10 @@
<exec osfamily="unix" dir="${auth.server.home}/bin" executable="./kc.sh" failonerror="true">
<arg value="build"/>
<arg value="--http-relative-path=/auth"/>
<arg value="--cache=local"/>
</exec>
<exec osfamily="windows" executable="${auth.server.home}/bin/kc.bat" failonerror="true">
<arg value="build"/>
<arg value="--http-relative-path=/auth"/>
<arg value="--cache=local"/>
</exec>
</target>

8
testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/AbstractQuarkusDeployableContainer.java
View file

@ -185,10 +185,12 @@ public abstract class AbstractQuarkusDeployableContainer implements DeployableCo
final String cacheMode = System.getProperty("auth.server.quarkus.cluster.config", "local");
if ("local".equals(cacheMode)) {
commands.add("--cache=local");
// Save ~2s for each Quarkus startup, when we know ISPN cluster is empty. See https://github.com/keycloak/keycloak/issues/21033
commands.add("-Djgroups.join_timeout=10");
} else {
commands.add("--cache=ispn");
commands.add("--cache-config-file=cluster-" + cacheMode + ".xml");
}
log.debugf("FIPS Mode: %s", configuration.getFipsMode());
@ -197,12 +199,6 @@ public abstract class AbstractQuarkusDeployableContainer implements DeployableCo
if (restart.get() || "ha".equals(cacheMode) || shouldSetUpDb.get() || configuration.getFipsMode() != FipsMode.DISABLED) {
prepareCommandsForRebuilding(commands);
if ("local".equals(cacheMode)) {
commands.add("--cache=local");
} else {
commands.add("--cache-config-file=cluster-" + cacheMode + ".xml");
}
if (configuration.getFipsMode() != FipsMode.DISABLED) {
addFipsOptions(commands);
}