Deprecate old remote store

Closes #32577

Signed-off-by: Pedro Ruivo <pruivo@redhat.com>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>
Co-authored-by: Alexander Schwartz <aschwart@redhat.com>

common/src/main/java/org/keycloak/common/Profile.java

@@ -120,7 +120,9 @@ public class Profile {
 
         ORGANIZATION("Organization support within realms", Type.PREVIEW),
 
-        PASSKEYS("Passkeys", Type.PREVIEW)
+        PASSKEYS("Passkeys", Type.PREVIEW),
+
+        REMOTE_STORE_CROSS_DC("Support for remote-store in embedded Infinispan caches", Type.DEPRECATED)
         ;
 
         private final Type type;

docs/documentation/upgrading/topics/changes/changes-26_0_0.adoc

@@ -111,14 +111,29 @@ As a consequence of the above changes, the following changes are required to you
 
 . `distributed-cache` definitions provided by a cache configuration file are ignored when the `multi-site` feature is enabled,
 so you must configure the connection to the external {jdgserver_name} deployment via the `cache-remote-*` command line arguments
-or Keycloak CR as outlined in the blueprints. If a `remote-store` configuration is detected in the cache configuration file,
+or Keycloak CR as outlined in the blueprints. All `remote-store` configurations must be removed from the cache configuration file.
-then a warning will be raised in the {project_name} logs.
 
 . Review your current cache configurations in the external {jdgserver_name} and update them with those outlined in the latest version of the {project_name}'s documentation.
 
 . While previous LoadBalancer configurations will continue to work with {project_name}, consider upgrading
 an existing Route53 configurations to avoid prolonged failover times due to client side DNS caching.
 
+= Deprecating remote-store in embedded Infinispan caches
+
+In {project_name} versions 24 to 25 to achieve a multi-site setup, a remote store in embedded Infinispan caches was configured.
+Manual configurations for the cache XML for multi-site were discouraged as CLI options to configure the caches are available.
+With the upgraded multi-site feature in 26, the same CLI options exist, but don't use embedded Infinispan caches anymore.
+
+In the community, remote stores for embedded Infinispan caches were used in some setups to keep user sessions when the {project_name} cluster was shut down or upgraded.
+This was never supported, documented or tested.
+As a fully supported alternative, the persistent user sessions feature should be used instead.
+
+Due to this, using remote caches in embedded Infinispan caches is now marked as deprecated with the plan to remove it.
+As it was never used outside multi-site setups that now achieve this by different means, it might be removed even in a future minor release.
+To be able to use the deprecated feature, you need to enable the feature `remote-store-cross-dc` or {project_name} will not start.
+
+There is an experimental feature `remote-cache` which allows leveraging the new multi-site mechanisms to store session related data in an external {jdgserver_name} server also for single site setups.
+
 = Admin Bootstrapping and Recovery
 
 It used to be difficult to regain access to a {project_name} instance when all admin users were locked out. The process required multiple advanced steps, including direct database access and manual changes. In an effort to improve the user experience, {project_name} now provides multiple ways to bootstrap a new admin account, which can be used to recover from such situations.

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/legacy/infinispan/CacheManagerFactory.java

@@ -298,15 +298,9 @@ public class CacheManagerFactory {
             // remove all distributed caches
             logger.debug("Removing all distributed caches.");
             for (String cacheName : CLUSTERED_CACHE_NAMES) {
-               var remoteStore = builders.get(cacheName)
+               if (hasRemoteStore(builders.get(cacheName))) {
-                     .persistence()
+                   logger.warnf("remote-store configuration detected for cache '%s'. Explicit cache configuration ignored when using '%s' or '%s' Features.", cacheName, Profile.Feature.REMOTE_CACHE.getKey(), Profile.Feature.MULTI_SITE.getKey());
-                     .stores()
+               }
-                     .stream()
-                     .filter(RemoteStoreConfigurationBuilder.class::isInstance)
-                     .findFirst();
-
-               if (remoteStore.isPresent())
-                  logger.warnf("remote-store configuration detected for cache '%s'. Explicit cache configuration ignored when using '%s' or '%s' Features.", cacheName, Profile.Feature.REMOTE_CACHE.getKey(), Profile.Feature.MULTI_SITE.getKey());
                builders.remove(cacheName);
             }
             // Disable JGroups, not required when the data is stored in the Remote Cache.
@@ -321,6 +315,8 @@ public class CacheManagerFactory {
             configureSessionsCaches(builder);
         }
 
+        checkForRemoteStores(builder);
+
         var start = isStartEagerly();
         return CompletableFuture.supplyAsync(() -> new DefaultCacheManager(builder, start));
     }
@@ -454,6 +450,26 @@ public class CacheManagerFactory {
         }
     }
 
+    private static void checkForRemoteStores(ConfigurationBuilderHolder builder) {
+        if (Profile.isFeatureEnabled(Profile.Feature.REMOTE_STORE_CROSS_DC) && Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE)) {
+            logger.fatalf("Feature %s is now deprecated.%nFor multi-site (cross-dc) support, enable only %s.",
+                    Profile.Feature.REMOTE_STORE_CROSS_DC.getKey(), Profile.Feature.MULTI_SITE.getKey());
+            throw new RuntimeException("The features " + Profile.Feature.REMOTE_STORE_CROSS_DC.getKey() + " and " + Profile.Feature.MULTI_SITE.getKey() + " must not be enabled at the same time.");
+        }
+        if (Profile.isFeatureEnabled(Profile.Feature.REMOTE_STORE_CROSS_DC) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+            logger.fatalf("Feature %s is now deprecated.%nFor multi-site (cross-dc) support, enable only %s.",
+                    Profile.Feature.REMOTE_STORE_CROSS_DC.getKey(), Profile.Feature.REMOTE_CACHE.getKey());
+            throw new RuntimeException("The features " + Profile.Feature.REMOTE_STORE_CROSS_DC.getKey() + " and " + Profile.Feature.REMOTE_CACHE.getKey() + " must not be enabled at the same time.");
+        }
+        if (!Profile.isFeatureEnabled(Profile.Feature.REMOTE_STORE_CROSS_DC)) {
+            if (builder.getNamedConfigurationBuilders().values().stream().anyMatch(CacheManagerFactory::hasRemoteStore)) {
+                logger.fatalf("Remote stores are not supported for embedded caches as feature %s is not enabled. This feature is disabled by default as it is now deprecated.%nFor keeping user sessions across restarts, use feature %s which is enabled by default.%nFor multi-site (cross-dc) support, enable %s.",
+                        Profile.Feature.REMOTE_STORE_CROSS_DC.getKey(), Profile.Feature.PERSISTENT_USER_SESSIONS.getKey(), Profile.Feature.MULTI_SITE.getKey());
+                throw new RuntimeException("Remote store is not supported as feature " + Profile.Feature.REMOTE_STORE_CROSS_DC.getKey() + " is not enabled.");
+            }
+        }
+    }
+
     private static void configureSessionsCaches(ConfigurationBuilderHolder builder) {
         Stream.of(USER_SESSION_CACHE_NAME, CLIENT_SESSION_CACHE_NAME, OFFLINE_USER_SESSION_CACHE_NAME, OFFLINE_CLIENT_SESSION_CACHE_NAME)
                 .forEach(cacheName -> {
@@ -487,4 +503,8 @@ public class CacheManagerFactory {
     private static String requiredStringProperty(String propertyName) {
         return Configuration.getOptionalKcValue(propertyName).orElseThrow(() -> new RuntimeException("Property " + propertyName + " required but not specified"));
     }
+
+    private static boolean hasRemoteStore(ConfigurationBuilder builder) {
+        return builder.persistence().stores().stream().anyMatch(RemoteStoreConfigurationBuilder.class::isInstance);
+    }
 }