External Infinispan as cache - Part 2

Includes a new implementation for the providers:

* StickySessionEncoderProviderFactory
* LoadBalancerCheckProviderFactory
* SingleUseObjectProviderFactory

Closes #28648

Signed-off-by: Pedro Ruivo <pruivo@redhat.com>

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java

@@ -29,13 +29,13 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ClusterProviderFactory;
-import org.keycloak.common.Profile;
 import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.DefaultInfinispanConnectionProviderFactory;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanUtil;
 import org.keycloak.connections.infinispan.TopologyInfo;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 
@@ -54,8 +54,6 @@ import java.util.stream.Collectors;
  */
 public class InfinispanClusterProviderFactory implements ClusterProviderFactory {
 
-    public static final String PROVIDER_ID = "infinispan";
-
     protected static final Logger logger = Logger.getLogger(InfinispanClusterProviderFactory.class);
 
     // Infinispan cache
@@ -185,12 +183,12 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public boolean isSupported(Config.Scope config) {
-        return !Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) || !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
+        return InfinispanUtils.isEmbeddedInfinispan();
     }
 
     @Listener

model/infinispan/src/main/java/org/keycloak/cluster/infinispan/remote/RemoteInfinispanClusterProviderFactory.java

@@ -9,11 +9,11 @@ import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ClusterProviderFactory;
 import org.keycloak.cluster.infinispan.InfinispanClusterProvider;
 import org.keycloak.cluster.infinispan.LockEntry;
-import org.keycloak.common.Profile;
 import org.keycloak.common.util.Retry;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.TopologyInfo;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 
@@ -26,13 +26,12 @@ import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.W
 
 public class RemoteInfinispanClusterProviderFactory implements ClusterProviderFactory {
 
-    public static final String PROVIDER_ID = "remote-infinispan";
     private static final Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
 
-    private RemoteCache<String, LockEntry> workCache;
+    private volatile RemoteCache<String, LockEntry> workCache;
-    private int clusterStartupTime;
+    private volatile int clusterStartupTime;
-    private RemoteInfinispanNotificationManager notificationManager;
+    private volatile RemoteInfinispanNotificationManager notificationManager;
-    private Executor executor;
+    private volatile Executor executor;
 
     @Override
     public ClusterProvider create(KeycloakSession session) {
@@ -75,12 +74,12 @@ public class RemoteInfinispanClusterProviderFactory implements ClusterProviderFa
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.REMOTE_PROVIDER_ID;
     }
 
     @Override
     public boolean isSupported(Config.Scope config) {
-        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
+        return InfinispanUtils.isRemoteInfinispan();
     }
 
     private static TopologyInfo getTopologyInfo(KeycloakSessionFactory factory) {

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProvider.java

@@ -17,14 +17,17 @@
 
 package org.keycloak.connections.infinispan;
 
+import java.util.Arrays;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 
 import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.infinispan.commons.util.concurrent.CompletionStages;
 import org.infinispan.factories.ComponentRegistry;
 import org.infinispan.factories.GlobalComponentRegistry;
+import org.infinispan.factories.KnownComponentNames;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.persistence.manager.PersistenceManager;
 import org.infinispan.util.concurrent.BlockingManager;
@@ -72,7 +75,7 @@ public class DefaultInfinispanConnectionProvider implements InfinispanConnection
         // Only the CacheStore (persistence) stores data in binary format and needs to be deleted.
         // We assume rolling-upgrade between KC 25 and KC 26 is not available, in other words, KC 25 and KC 26 servers are not present in the same cluster.
         var stage = CompletionStages.aggregateCompletionStage();
-        DISTRIBUTED_REPLICATED_CACHE_NAMES.stream()
+        Arrays.stream(CLUSTERED_CACHE_NAMES)
                 .map(this::getCache)
                 .map(DefaultInfinispanConnectionProvider::persistenceManager)
                 .map(DefaultInfinispanConnectionProvider::clearPersistenceManager)
@@ -85,6 +88,12 @@ public class DefaultInfinispanConnectionProvider implements InfinispanConnection
         return GlobalComponentRegistry.componentOf(cacheManager, BlockingManager.class).asExecutor(name);
     }
 
+    @Override
+    public ScheduledExecutorService getScheduledExecutor() {
+        //noinspection removal
+        return GlobalComponentRegistry.of(cacheManager).getComponent(ScheduledExecutorService.class, KnownComponentNames.TIMEOUT_SCHEDULE_EXECUTOR);
+    }
+
     @Override
     public void close() {
     }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -17,6 +17,7 @@
 
 package org.keycloak.connections.infinispan;
 
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
@@ -44,9 +45,8 @@ import org.keycloak.Config;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.cluster.ManagedCacheManagerProvider;
-import org.keycloak.common.Profile;
 import org.keycloak.connections.infinispan.remote.RemoteInfinispanConnectionProvider;
-import org.keycloak.marshalling.KeycloakModelSchema;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.marshalling.Marshalling;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
@@ -64,7 +64,7 @@ import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.A
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME;
-import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.DISTRIBUTED_REPLICATED_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME;
@@ -107,11 +107,10 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
     public InfinispanConnectionProvider create(KeycloakSession session) {
         lazyInit();
 
-        if (Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+        return InfinispanUtils.isRemoteInfinispan() ?
-            return new RemoteInfinispanConnectionProvider(cacheManager, remoteCacheManager, topologyInfo);
+                new RemoteInfinispanConnectionProvider(cacheManager, remoteCacheManager, topologyInfo) :
-        }
+                new DefaultInfinispanConnectionProvider(cacheManager, remoteCacheProvider, topologyInfo);
 
-        return new DefaultInfinispanConnectionProvider(cacheManager, remoteCacheProvider, topologyInfo);
     }
 
     /*
@@ -202,7 +201,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                         }
 
                         managedCacheManager = provider.getEmbeddedCacheManager(config);
-                        if (Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+                        if (InfinispanUtils.isRemoteInfinispan()) {
                             rcm = provider.getRemoteCacheManager(config);
                         }
                     }
@@ -214,7 +213,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                             throw new RuntimeException("No " + ManagedCacheManagerProvider.class.getName() + " found. If running in embedded mode set the [embedded] property to this provider.");
                         }
                         localCacheManager = initEmbedded();
-                        if (Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+                        if (InfinispanUtils.isRemoteInfinispan()) {
                             rcm = initRemote();
                         }
                     } else {
@@ -245,7 +244,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         RemoteCacheManager remoteCacheManager = new RemoteCacheManager(builder.build());
 
         // establish connection to all caches
-        DISTRIBUTED_REPLICATED_CACHE_NAMES.forEach(remoteCacheManager::getCache);
+        Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(remoteCacheManager::getCache);
         return remoteCacheManager;
 
     }
@@ -342,17 +341,16 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         defineClusteredCache(cacheManager, OFFLINE_CLIENT_SESSION_CACHE_NAME, clusteredConfiguration);
         defineClusteredCache(cacheManager, LOGIN_FAILURE_CACHE_NAME, clusteredConfiguration);
 
-        var actionTokenBuilder = getActionTokenCacheConfig();
+        if (InfinispanUtils.isEmbeddedInfinispan()) {
-        if (clustered) {
-            actionTokenBuilder.simpleCache(false);
-            actionTokenBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
-        }
-        defineClusteredCache(cacheManager, ACTION_TOKEN_CACHE, actionTokenBuilder.build());
-
-
-        if (!Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) || !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
             defineClusteredCache(cacheManager, AUTHENTICATION_SESSIONS_CACHE_NAME, clusteredConfiguration);
 
+            var actionTokenBuilder = getActionTokenCacheConfig();
+            if (clustered) {
+                actionTokenBuilder.simpleCache(false);
+                actionTokenBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
+            }
+            defineClusteredCache(cacheManager, ACTION_TOKEN_CACHE, actionTokenBuilder.build());
+
             var workBuilder = createCacheConfigurationBuilder()
                     .expiration().enableReaper().wakeUpInterval(15, TimeUnit.SECONDS);
             if (clustered) {

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -17,9 +17,11 @@
 
 package org.keycloak.connections.infinispan;
 
-import java.util.List;
+import java.util.Arrays;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.stream.Stream;
 
 import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
@@ -72,26 +74,19 @@ public interface InfinispanConnectionProvider extends Provider {
     // Constant used as the prefix of the current node if "jboss.node.name" is not configured
     String NODE_PREFIX = "node_";
 
-    String[] ALL_CACHES_NAME = {
+    // list of cache name for local caches (not replicated)
+    String[] LOCAL_CACHE_NAMES = {
             REALM_CACHE_NAME,
             REALM_REVISIONS_CACHE_NAME,
             USER_CACHE_NAME,
             USER_REVISIONS_CACHE_NAME,
-            USER_SESSION_CACHE_NAME,
-            CLIENT_SESSION_CACHE_NAME,
-            OFFLINE_USER_SESSION_CACHE_NAME,
-            OFFLINE_CLIENT_SESSION_CACHE_NAME,
-            LOGIN_FAILURE_CACHE_NAME,
-            AUTHENTICATION_SESSIONS_CACHE_NAME,
-            WORK_CACHE_NAME,
             AUTHORIZATION_CACHE_NAME,
             AUTHORIZATION_REVISIONS_CACHE_NAME,
-            ACTION_TOKEN_CACHE,
             KEYS_CACHE_NAME
     };
 
     // list of cache name which could be defined as distributed or replicated
-    public static List<String> DISTRIBUTED_REPLICATED_CACHE_NAMES = List.of(
+    String[] CLUSTERED_CACHE_NAMES = {
             USER_SESSION_CACHE_NAME,
             CLIENT_SESSION_CACHE_NAME,
             OFFLINE_USER_SESSION_CACHE_NAME,
@@ -99,7 +94,10 @@ public interface InfinispanConnectionProvider extends Provider {
             LOGIN_FAILURE_CACHE_NAME,
             AUTHENTICATION_SESSIONS_CACHE_NAME,
             ACTION_TOKEN_CACHE,
-            WORK_CACHE_NAME);
+            WORK_CACHE_NAME
+    };
+
+    String[] ALL_CACHES_NAME = Stream.concat(Arrays.stream(LOCAL_CACHE_NAMES), Arrays.stream(CLUSTERED_CACHE_NAMES)).toArray(String[]::new);
 
     /**
      *
@@ -150,6 +148,11 @@ public interface InfinispanConnectionProvider extends Provider {
      */
     Executor getExecutor(String name);
 
+    /**
+     * @return The Infinispan {@link ScheduledExecutorService}. Long or blocking operations must not be executed directly.
+     */
+    ScheduledExecutorService getScheduledExecutor();
+
     /**
      * Syntactic sugar to get a {@link RemoteCache}.
      *

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanMultiSiteLoadBalancerCheckProviderFactory.java

@@ -17,6 +17,8 @@
 
 package org.keycloak.connections.infinispan;
 
+import org.infinispan.factories.ComponentRegistry;
+import org.infinispan.persistence.manager.PersistenceManager;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.common.Profile;
@@ -26,15 +28,13 @@ import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.provider.EnvironmentDependentProviderFactory;
 
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ALL_CACHES_NAME;
 
 
 public class InfinispanMultiSiteLoadBalancerCheckProviderFactory implements LoadBalancerCheckProviderFactory, EnvironmentDependentProviderFactory {
 
     private LoadBalancerCheckProvider loadBalancerCheckProvider;
-    private static final LoadBalancerCheckProvider ALWAYS_HEALTHY = new LoadBalancerCheckProvider() {
+    public static final LoadBalancerCheckProvider ALWAYS_HEALTHY = () -> false;
-        @Override public boolean isDown() { return false; }
-        @Override public void close() {}
-    };
     private static final Logger LOG = Logger.getLogger(InfinispanMultiSiteLoadBalancerCheckProviderFactory.class);
 
     @Override
@@ -45,7 +45,7 @@ public class InfinispanMultiSiteLoadBalancerCheckProviderFactory implements Load
                 LOG.warn("InfinispanConnectionProvider is not available. Load balancer check will be always healthy for Infinispan.");
                 loadBalancerCheckProvider = ALWAYS_HEALTHY;
             } else {
-                loadBalancerCheckProvider = new InfinispanMultiSiteLoadBalancerCheckProvider(infinispanConnectionProvider);
+                loadBalancerCheckProvider = () -> isEmbeddedCachesDown(infinispanConnectionProvider);
             }
         }
         return loadBalancerCheckProvider;
@@ -73,6 +73,29 @@ public class InfinispanMultiSiteLoadBalancerCheckProviderFactory implements Load
 
     @Override
     public boolean isSupported(Config.Scope config) {
-        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE);
+        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
+    }
+
+    private boolean isEmbeddedCachesDown(InfinispanConnectionProvider provider) {
+        return isAnyEmbeddedCachesDown(provider, ALL_CACHES_NAME, LOG);
+    }
+
+    public static boolean isAnyEmbeddedCachesDown(InfinispanConnectionProvider connectionProvider, String[] cacheNames, Logger logger) {
+        for (var name : cacheNames) {
+            var cache = connectionProvider.getCache(name, false);
+
+            // check if cache is started
+            if (cache == null || !cache.getStatus().allowInvocations()) {
+                logger.debugf("Cache '%s' is not started yet.", name);
+                return true; // no need to check other caches
+            }
+
+            var persistenceManager = ComponentRegistry.componentOf(cache, PersistenceManager.class);
+            if (persistenceManager != null && !persistenceManager.isAvailable()) {
+                logger.debugf("Persistence for embedded cache '%s' is down.", name);
+                return true; // no need to check other caches
+            }
+        }
+        return false;
     }
 }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/RemoteCacheProvider.java

@@ -96,10 +96,12 @@ public class RemoteCacheProvider {
     protected synchronized RemoteCache loadRemoteCache(String cacheName) {
         RemoteCache remoteCache = InfinispanUtil.getRemoteCache(cacheManager.getCache(cacheName));
 
-        if (remoteCache != null) {
+        if (remoteCache == null) {
-            logger.infof("Hotrod version for remoteCache %s: %s", remoteCache.getName(), remoteCache.getRemoteCacheManager().getConfiguration().version());
+            return null;
         }
 
+        logger.infof("Hotrod version for remoteCache %s: %s", remoteCache.getName(), remoteCache.getRemoteCacheManager().getConfiguration().version());
+
         Boolean remoteStoreSecurity = config.getBoolean("remoteStoreSecurityEnabled");
         if (remoteStoreSecurity == null) {
             try {

model/infinispan/src/main/java/org/keycloak/connections/infinispan/remote/RemoteInfinispanConnectionProvider.java

@@ -1,14 +1,17 @@
 package org.keycloak.connections.infinispan.remote;
 
+import java.util.Arrays;
 import java.util.Objects;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 
 import org.infinispan.Cache;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.infinispan.client.hotrod.RemoteCacheManager;
 import org.infinispan.commons.util.concurrent.CompletionStages;
 import org.infinispan.factories.GlobalComponentRegistry;
+import org.infinispan.factories.KnownComponentNames;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.util.concurrent.BlockingManager;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
@@ -44,7 +47,7 @@ public record RemoteInfinispanConnectionProvider(EmbeddedCacheManager embeddedCa
         // Only the CacheStore (persistence) stores data in binary format and needs to be deleted.
         // We assume rolling-upgrade between KC 25 and KC 26 is not available, in other words, KC 25 and KC 26 servers are not present in the same cluster.
         var stage = CompletionStages.aggregateCompletionStage();
-        DISTRIBUTED_REPLICATED_CACHE_NAMES.stream()
+        Arrays.stream(CLUSTERED_CACHE_NAMES)
                 .map(this::getRemoteCache)
                 .map(RemoteCache::clearAsync)
                 .forEach(stage::dependsOn);
@@ -56,6 +59,12 @@ public record RemoteInfinispanConnectionProvider(EmbeddedCacheManager embeddedCa
         return GlobalComponentRegistry.componentOf(embeddedCacheManager, BlockingManager.class).asExecutor(name);
     }
 
+    @Override
+    public ScheduledExecutorService getScheduledExecutor() {
+        //noinspection removal
+        return GlobalComponentRegistry.of(embeddedCacheManager).getComponent(ScheduledExecutorService.class, KnownComponentNames.TIMEOUT_SCHEDULE_EXECUTOR);
+    }
+
     @Override
     public void close() {
         //no-op

model/infinispan/src/main/java/org/keycloak/connections/infinispan/remote/RemoteLoadBalancerCheckProviderFactory.java

@@ -0,0 +1,179 @@
+package org.keycloak.connections.infinispan.remote;
+
+import org.infinispan.client.hotrod.impl.InternalRemoteCache;
+import org.infinispan.client.hotrod.impl.operations.PingResponse;
+import org.infinispan.commons.util.concurrent.CompletableFutures;
+import org.infinispan.util.concurrent.ActionSequencer;
+import org.jboss.logging.Logger;
+import org.keycloak.Config;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.health.LoadBalancerCheckProvider;
+import org.keycloak.health.LoadBalancerCheckProviderFactory;
+import org.keycloak.infinispan.util.InfinispanUtils;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.provider.EnvironmentDependentProviderFactory;
+import org.keycloak.provider.ProviderConfigProperty;
+import org.keycloak.provider.ProviderConfigurationBuilder;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanMultiSiteLoadBalancerCheckProviderFactory.ALWAYS_HEALTHY;
+import static org.keycloak.connections.infinispan.InfinispanMultiSiteLoadBalancerCheckProviderFactory.isAnyEmbeddedCachesDown;
+
+public class RemoteLoadBalancerCheckProviderFactory implements LoadBalancerCheckProviderFactory, EnvironmentDependentProviderFactory {
+
+    private static final int DEFAULT_POLL_INTERVAL = 5000;
+    private static final Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+
+    private volatile int pollIntervalMillis;
+    private volatile LoadBalancerCheckProvider provider;
+    private InfinispanConnectionProvider connectionProvider;
+    private ScheduledFuture<?> availabilityFuture;
+    private RemoteCacheCheckList remoteCacheCheckList;
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        return InfinispanUtils.isRemoteInfinispan();
+    }
+
+    @Override
+    public LoadBalancerCheckProvider create(KeycloakSession session) {
+        return provider;
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+        pollIntervalMillis = config.getInt("poll-interval", DEFAULT_POLL_INTERVAL);
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+        try (var session = factory.create()) {
+            var provider = session.getProvider(InfinispanConnectionProvider.class);
+            if (provider == null) {
+                logger.warn("InfinispanConnectionProvider is not available. Load balancer check will be always healthy for Infinispan.");
+                this.provider = ALWAYS_HEALTHY;
+                return;
+            }
+            this.connectionProvider = provider;
+
+            var remoteCacheChecks = Arrays.stream(CLUSTERED_CACHE_NAMES)
+                    .map(s -> new RemoteCacheCheck(s, provider))
+                    .collect(Collectors.toList());
+            var sequencer = new ActionSequencer(connectionProvider.getExecutor("load-balancer-check"), false, null);
+
+            this.remoteCacheCheckList = new RemoteCacheCheckList(remoteCacheChecks, sequencer);
+            this.availabilityFuture = provider.getScheduledExecutor()
+                    .scheduleAtFixedRate(remoteCacheCheckList, pollIntervalMillis, pollIntervalMillis, TimeUnit.MILLISECONDS);
+
+            this.provider = this::isAnyCacheDown;
+        }
+    }
+
+    @Override
+    public void close() {
+        if (availabilityFuture != null) {
+            availabilityFuture.cancel(true);
+            availabilityFuture = null;
+        }
+        provider = null;
+        remoteCacheCheckList = null;
+    }
+
+    @Override
+    public String getId() {
+        return InfinispanUtils.REMOTE_PROVIDER_ID;
+    }
+
+    @Override
+    public int order() {
+        return InfinispanUtils.PROVIDER_ORDER;
+    }
+
+    @Override
+    public List<ProviderConfigProperty> getConfigMetadata() {
+        return ProviderConfigurationBuilder.create()
+                .property()
+                .name("poll-interval")
+                .type("int")
+                .helpText("The Remote caches poll interval, in milliseconds, for connection availability")
+                .defaultValue(DEFAULT_POLL_INTERVAL)
+                .add()
+                .build();
+    }
+
+    private boolean isAnyCacheDown() {
+        return isEmbeddedCachesDown() || remoteCacheCheckList.isDown();
+    }
+
+    private boolean isEmbeddedCachesDown() {
+        return isAnyEmbeddedCachesDown(connectionProvider, LOCAL_CACHE_NAMES, logger);
+    }
+
+    private record RemoteCacheCheckList(List<RemoteCacheCheck> list, ActionSequencer sequencer) implements Runnable {
+        @Override
+        public void run() {
+            list.forEach(remoteCacheCheck -> sequencer.orderOnKey(remoteCacheCheck.name(), remoteCacheCheck));
+        }
+
+        public boolean isDown() {
+            return list.stream().anyMatch(RemoteCacheCheck::isDown);
+        }
+    }
+
+    private static class RemoteCacheCheck implements Callable<CompletionStage<Void>>, BiFunction<PingResponse, Throwable, Void> {
+
+        private final String name;
+        private final InfinispanConnectionProvider provider;
+        private volatile boolean isDown;
+
+        private RemoteCacheCheck(String name, InfinispanConnectionProvider provider) {
+            this.name = name;
+            this.provider = provider;
+        }
+
+        String name() {
+            return name;
+        }
+
+        boolean isDown() {
+            return isDown;
+        }
+
+        @Override
+        public CompletionStage<Void> call() {
+            try {
+                var cache = provider.getRemoteCache(name);
+                if (cache instanceof InternalRemoteCache<Object, Object>) {
+                    return ((InternalRemoteCache<Object, Object>) cache).ping()
+                            .handle(this);
+                }
+                isDown = false;
+            } catch (Exception e) {
+                if (!isDown) {
+                    logger.warnf("Remote cache '%' is down.", name);
+                }
+                isDown = true;
+            }
+            return CompletableFutures.completedNull();
+        }
+
+        @Override
+        public Void apply(PingResponse response, Throwable throwable) {
+            logger.debugf("Received Ping response for cache '%s'. Success=%s, Throwable=%s", name, response.isSuccess(), throwable);
+            isDown = throwable != null || !response.isSuccess();
+            return null;
+        }
+    }
+}

model/infinispan/src/main/java/org/keycloak/infinispan/util/InfinispanUtils.java

@@ -0,0 +1,32 @@
+package org.keycloak.infinispan.util;
+
+import org.keycloak.common.Profile;
+import org.keycloak.common.Profile.Feature;
+
+import static org.keycloak.common.Profile.Feature.MULTI_SITE;
+import static org.keycloak.common.Profile.Feature.REMOTE_CACHE;
+
+public final class InfinispanUtils {
+
+    private InfinispanUtils() {
+    }
+
+    // all providers have the same order
+    public static final int PROVIDER_ORDER = 1;
+
+    // provider id for embedded cache providers
+    public static final String EMBEDDED_PROVIDER_ID = "infinispan";
+
+    // provider id for remote cache providers
+    public static final String REMOTE_PROVIDER_ID = "remote";
+
+    // true if running with external infinispan mode only
+    public static boolean isRemoteInfinispan() {
+        return Profile.isFeatureEnabled(Feature.MULTI_SITE) && Profile.isFeatureEnabled(REMOTE_CACHE);
+    }
+
+    // true if running with embedded caches.
+    public static boolean isEmbeddedInfinispan() {
+        return !Profile.isFeatureEnabled(MULTI_SITE) || !Profile.isFeatureEnabled(REMOTE_CACHE);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanAuthenticationSessionProviderFactory.java

@@ -22,8 +22,8 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterProvider;
-import org.keycloak.common.Profile;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.cache.infinispan.events.AuthenticationSessionAuthNoteUpdateEvent;
@@ -51,7 +51,6 @@ import java.util.concurrent.ConcurrentHashMap;
 public class InfinispanAuthenticationSessionProviderFactory implements AuthenticationSessionProviderFactory<InfinispanAuthenticationSessionProvider> {
 
     private static final Logger log = Logger.getLogger(InfinispanAuthenticationSessionProviderFactory.class);
-    public static final int PROVIDER_PRIORITY = 1;
 
     private InfinispanKeyGenerator keyGenerator;
 
@@ -59,8 +58,6 @@ public class InfinispanAuthenticationSessionProviderFactory implements Authentic
 
     private int authSessionsLimit;
 
-    public static final String PROVIDER_ID = "infinispan";
-
     public static final String AUTH_SESSIONS_LIMIT = "authSessionsLimit";
 
     public static final int DEFAULT_AUTH_SESSIONS_LIMIT = 300;
@@ -191,16 +188,16 @@ public class InfinispanAuthenticationSessionProviderFactory implements Authentic
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        return PROVIDER_PRIORITY;
+        return InfinispanUtils.PROVIDER_ORDER;
     }
 
     @Override
     public boolean isSupported(Config.Scope config) {
-        return !Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) || !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
+        return InfinispanUtils.isEmbeddedInfinispan();
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanSingleUseObjectProviderFactory.java

@@ -26,6 +26,7 @@ import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanUtil;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.SingleUseObjectProviderFactory;
@@ -33,8 +34,6 @@ import org.keycloak.models.sessions.infinispan.entities.SingleUseObjectValueEnti
 
 import java.util.function.Supplier;
 
-import static org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory.PROVIDER_PRIORITY;
-
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
@@ -85,11 +84,16 @@ public class InfinispanSingleUseObjectProviderFactory implements SingleUseObject
 
     @Override
     public String getId() {
-        return "infinispan";
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        return PROVIDER_PRIORITY;
+        return InfinispanUtils.PROVIDER_ORDER;
+    }
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        return InfinispanUtils.isEmbeddedInfinispan();
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanStickySessionEncoderProviderFactory.java

@@ -19,13 +19,13 @@ package org.keycloak.models.sessions.infinispan;
 
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.provider.ProviderConfigProperty;
 import org.keycloak.provider.ProviderConfigurationBuilder;
 import org.keycloak.sessions.StickySessionEncoderProvider;
 import org.keycloak.sessions.StickySessionEncoderProviderFactory;
-import static org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory.PROVIDER_PRIORITY;
 
 import java.util.List;
 
@@ -36,7 +36,6 @@ public class InfinispanStickySessionEncoderProviderFactory implements StickySess
 
     private static final Logger log = Logger.getLogger(InfinispanStickySessionEncoderProviderFactory.class);
 
-
     private boolean shouldAttachRoute;
 
     @Override
@@ -46,14 +45,14 @@ public class InfinispanStickySessionEncoderProviderFactory implements StickySess
 
     @Override
     public void init(Config.Scope config) {
-        this.shouldAttachRoute = config.getBoolean("shouldAttachRoute", true);
+        setShouldAttachRoute(config.getBoolean("shouldAttachRoute", true));
-        log.debugf("Should attach route to the sticky session cookie: %b", shouldAttachRoute);
-
     }
 
     // Used for testing
+    @Override
     public void setShouldAttachRoute(boolean shouldAttachRoute) {
         this.shouldAttachRoute = shouldAttachRoute;
+        log.debugf("Should attach route to the sticky session cookie: %b", shouldAttachRoute);
     }
 
     @Override
@@ -68,12 +67,12 @@ public class InfinispanStickySessionEncoderProviderFactory implements StickySess
 
     @Override
     public String getId() {
-        return "infinispan";
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        return PROVIDER_PRIORITY;
+        return InfinispanUtils.PROVIDER_ORDER;
     }
 
     @Override
@@ -87,4 +86,9 @@ public class InfinispanStickySessionEncoderProviderFactory implements StickySess
                 .add()
                 .build();
     }
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        return InfinispanUtils.isEmbeddedInfinispan();
+    }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserLoginFailureProviderFactory.java

@@ -22,10 +22,10 @@ import org.infinispan.persistence.remote.RemoteStore;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
 import org.keycloak.cluster.ClusterProvider;
-import org.keycloak.common.Profile;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanUtil;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.KeycloakSessionTask;
@@ -52,8 +52,6 @@ import org.keycloak.models.utils.PostMigrationEvent;
 
 import java.util.Set;
 
-import static org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory.PROVIDER_PRIORITY;
-
 /**
  * @author <a href="mailto:mkanis@redhat.com">Martin Kanis</a>
  */
@@ -61,8 +59,6 @@ public class InfinispanUserLoginFailureProviderFactory implements UserLoginFailu
 
     private static final Logger log = Logger.getLogger(InfinispanUserLoginFailureProviderFactory.class);
 
-    public static final String PROVIDER_ID = "infinispan";
-
     public static final String REALM_REMOVED_SESSION_EVENT = "REALM_REMOVED_EVENT_SESSIONS";
 
     public static final String REMOVE_ALL_LOGIN_FAILURES_EVENT = "REMOVE_ALL_LOGIN_FAILURES_EVENT";
@@ -95,7 +91,7 @@ public class InfinispanUserLoginFailureProviderFactory implements UserLoginFailu
                     checkRemoteCaches(session);
                     registerClusterListeners(session);
                     // TODO [pruivo] to remove: workaround to run the testsuite.
-                    if (!Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) || !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+                    if (InfinispanUtils.isEmbeddedInfinispan()) {
                         loadLoginFailuresFromRemoteCaches(session);
                     }
                 });
@@ -220,11 +216,11 @@ public class InfinispanUserLoginFailureProviderFactory implements UserLoginFailu
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        return PROVIDER_PRIORITY;
+        return InfinispanUtils.PROVIDER_ORDER;
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java

@@ -36,6 +36,7 @@ import org.keycloak.common.util.Environment;
 import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanUtil;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.ClientModel;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
@@ -74,14 +75,10 @@ import org.keycloak.provider.ProviderEvent;
 import org.keycloak.provider.ProviderEventListener;
 import org.keycloak.provider.ServerInfoAwareProviderFactory;
 
-import static org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory.PROVIDER_PRIORITY;
-
 public class InfinispanUserSessionProviderFactory implements UserSessionProviderFactory, ServerInfoAwareProviderFactory {
 
     private static final Logger log = Logger.getLogger(InfinispanUserSessionProviderFactory.class);
 
-    public static final String PROVIDER_ID = "infinispan";
-
     public static final String REALM_REMOVED_SESSION_EVENT = "REALM_REMOVED_EVENT_SESSIONS";
 
     public static final String REMOVE_USER_SESSIONS_EVENT = "REMOVE_USER_SESSIONS_EVENT";
@@ -183,7 +180,7 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
                         }
                         registerClusterListeners(session);
                         // TODO [pruivo] to remove: workaround to run the testsuite.
-                        if (!Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) || !Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE)) {
+                        if (InfinispanUtils.isEmbeddedInfinispan()) {
                             loadSessionsFromRemoteCaches(session);
                         }
 
@@ -424,12 +421,12 @@ public class InfinispanUserSessionProviderFactory implements UserSessionProvider
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.EMBEDDED_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        return PROVIDER_PRIORITY;
+        return InfinispanUtils.PROVIDER_ORDER;
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/events/AbstractAuthSessionClusterListener.java

@@ -20,10 +20,10 @@ package org.keycloak.models.sessions.infinispan.events;
 import org.jboss.logging.Logger;
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.cluster.ClusterListener;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProvider;
-import org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.sessions.AuthenticationSessionProvider;
 
@@ -45,7 +45,7 @@ public abstract class AbstractAuthSessionClusterListener <SE extends SessionClus
     public void eventReceived(ClusterEvent event) {
         KeycloakModelUtils.runJobInTransaction(sessionFactory, (KeycloakSession session) -> {
             InfinispanAuthenticationSessionProvider provider = (InfinispanAuthenticationSessionProvider) session.getProvider(AuthenticationSessionProvider.class,
-                    InfinispanAuthenticationSessionProviderFactory.PROVIDER_ID);
+                    InfinispanUtils.EMBEDDED_PROVIDER_ID);
             SE sessionEvent = (SE) event;
 
             if (!provider.getCache().getStatus().allowInvocations()) {

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remote/RemoteInfinispanAuthenticationSessionProviderFactory.java

@@ -6,7 +6,7 @@ import java.util.List;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.jboss.logging.Logger;
 import org.keycloak.Config;
-import org.keycloak.common.Profile;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory;
@@ -22,14 +22,13 @@ import static org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSe
 public class RemoteInfinispanAuthenticationSessionProviderFactory implements AuthenticationSessionProviderFactory<RemoteInfinispanAuthenticationSessionProvider> {
 
     private final static Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
-    public static final String PROVIDER_ID = "remote-infinispan";
 
     private int authSessionsLimit;
-    private RemoteCache<String, RootAuthenticationSessionEntity> cache;
+    private volatile RemoteCache<String, RootAuthenticationSessionEntity> cache;
 
     @Override
     public boolean isSupported(Config.Scope config) {
-        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE) && Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
+        return InfinispanUtils.isRemoteInfinispan();
     }
 
     @Override
@@ -67,13 +66,12 @@ public class RemoteInfinispanAuthenticationSessionProviderFactory implements Aut
 
     @Override
     public String getId() {
-        return PROVIDER_ID;
+        return InfinispanUtils.REMOTE_PROVIDER_ID;
     }
 
     @Override
     public int order() {
-        // use the same priority as the embedded based one
+        return InfinispanUtils.PROVIDER_ORDER;
-        return InfinispanAuthenticationSessionProviderFactory.PROVIDER_PRIORITY;
     }
 
     public int getAuthSessionsLimit() {

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remote/RemoteInfinispanKeycloakTransaction.java

@@ -75,7 +75,7 @@ public class RemoteInfinispanKeycloakTransaction<K, V> implements KeycloakTransa
         return active;
     }
 
-    public void put(K key, V value, int lifespan, TimeUnit timeUnit) {
+    public void put(K key, V value, long lifespan, TimeUnit timeUnit) {
         logger.tracef("Adding %s.put(%S)", cache.getName(), key);
 
         if (tasks.containsKey(key)) {
@@ -163,7 +163,7 @@ public class RemoteInfinispanKeycloakTransaction<K, V> implements KeycloakTransa
         }
     }
 
-    private record PutOperation<K, V>(K key, V value, int lifespan, TimeUnit timeUnit) implements Operation<K, V> {
+    private record PutOperation<K, V>(K key, V value, long lifespan, TimeUnit timeUnit) implements Operation<K, V> {
 
         @Override
         public CompletionStage<?> execute(RemoteCache<K, V> cache) {
@@ -194,7 +194,7 @@ public class RemoteInfinispanKeycloakTransaction<K, V> implements KeycloakTransa
 
     }
 
-    private record ReplaceOperation<K, V>(K key, V value, int lifespan, TimeUnit timeUnit) implements Operation<K, V> {
+    private record ReplaceOperation<K, V>(K key, V value, long lifespan, TimeUnit timeUnit) implements Operation<K, V> {
 
         @Override
         public CompletionStage<?> execute(RemoteCache<K, V> cache) {

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remote/RemoteInfinispanSingleUseObjectProvider.java

@@ -0,0 +1,86 @@
+package org.keycloak.models.sessions.infinispan.remote;
+
+import org.infinispan.client.hotrod.Flag;
+import org.infinispan.client.hotrod.RemoteCache;
+import org.infinispan.client.hotrod.exceptions.HotRodClientException;
+import org.jboss.logging.Logger;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.SingleUseObjectProvider;
+import org.keycloak.models.sessions.infinispan.entities.SingleUseObjectValueEntity;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class RemoteInfinispanSingleUseObjectProvider implements SingleUseObjectProvider {
+
+    private final static Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+
+    private final RemoteInfinispanKeycloakTransaction<String, SingleUseObjectValueEntity> transaction;
+
+    public RemoteInfinispanSingleUseObjectProvider(KeycloakSession session, RemoteCache<String, SingleUseObjectValueEntity> cache) {
+        transaction = new RemoteInfinispanKeycloakTransaction<>(cache);
+        session.getTransactionManager().enlistAfterCompletion(transaction);
+    }
+
+    @Override
+    public void put(String key, long lifespanSeconds, Map<String, String> notes) {
+        transaction.put(key, wrap(notes), lifespanSeconds, TimeUnit.SECONDS);
+    }
+
+    @Override
+    public Map<String, String> get(String key) {
+        return unwrap(transaction.get(key));
+    }
+
+    @Override
+    public Map<String, String> remove(String key) {
+        try {
+            return unwrap(withReturnValue().remove(key));
+        } catch (HotRodClientException re) {
+            // No need to retry. The hotrod (remoteCache) has some retries in itself in case of some random network error happened.
+            // In case of lock conflict, we don't want to retry anyway as there was likely an attempt to remove the code from different place.
+            logger.debugf(re, "Failed when removing code %s", key);
+            return null;
+        }
+    }
+
+    @Override
+    public boolean replace(String key, Map<String, String> notes) {
+        return withReturnValue().replace(key, wrap(notes)) != null;
+    }
+
+    @Override
+    public boolean putIfAbsent(String key, long lifespanInSeconds) {
+        try {
+            return withReturnValue().putIfAbsent(key, wrap(null), lifespanInSeconds, TimeUnit.SECONDS) == null;
+        } catch (HotRodClientException re) {
+            // No need to retry. The hotrod (remoteCache) has some retries in itself in case of some random network error happened.
+            // In case of lock conflict, we don't want to retry anyway as there was likely an attempt to use the token from different place.
+            logger.debugf(re, "Failed when adding token %s", key);
+            return false;
+        }
+    }
+
+    @Override
+    public boolean contains(String key) {
+        return transaction.getCache().containsKey(key);
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    private RemoteCache<String, SingleUseObjectValueEntity> withReturnValue() {
+        return transaction.getCache().withFlags(Flag.FORCE_RETURN_VALUE);
+    }
+
+    private static Map<String, String> unwrap(SingleUseObjectValueEntity entity) {
+        return entity == null ? null : entity.getNotes();
+    }
+
+    private static SingleUseObjectValueEntity wrap(Map<String, String> notes) {
+        return new SingleUseObjectValueEntity(notes);
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remote/RemoteInfinispanSingleUseObjectProviderFactory.java

@@ -0,0 +1,59 @@
+package org.keycloak.models.sessions.infinispan.remote;
+
+import org.infinispan.client.hotrod.RemoteCache;
+import org.jboss.logging.Logger;
+import org.keycloak.Config;
+import org.keycloak.infinispan.util.InfinispanUtils;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.models.SingleUseObjectProviderFactory;
+import org.keycloak.models.sessions.infinispan.entities.SingleUseObjectValueEntity;
+
+import java.lang.invoke.MethodHandles;
+
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ACTION_TOKEN_CACHE;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.getRemoteCache;
+
+public class RemoteInfinispanSingleUseObjectProviderFactory implements SingleUseObjectProviderFactory<RemoteInfinispanSingleUseObjectProvider> {
+
+    private final static Logger logger = Logger.getLogger(MethodHandles.lookup().lookupClass());
+
+    private volatile RemoteCache<String, SingleUseObjectValueEntity> cache;
+
+    @Override
+    public RemoteInfinispanSingleUseObjectProvider create(KeycloakSession session) {
+        assert cache != null;
+        return new RemoteInfinispanSingleUseObjectProvider(session, cache);
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+        cache = getRemoteCache(factory, ACTION_TOKEN_CACHE);
+        logger.debug("Provided initialized.");
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public String getId() {
+        return InfinispanUtils.REMOTE_PROVIDER_ID;
+    }
+
+    @Override
+    public int order() {
+        return InfinispanUtils.PROVIDER_ORDER;
+    }
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        return InfinispanUtils.isRemoteInfinispan();
+    }
+}

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remote/RemoteStickySessionEncoderProviderFactory.java

@@ -0,0 +1,130 @@
+package org.keycloak.models.sessions.infinispan.remote;
+
+import org.jboss.logging.Logger;
+import org.keycloak.Config;
+import org.keycloak.connections.infinispan.InfinispanUtil;
+import org.keycloak.infinispan.util.InfinispanUtils;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.provider.ProviderConfigProperty;
+import org.keycloak.provider.ProviderConfigurationBuilder;
+import org.keycloak.sessions.StickySessionEncoderProvider;
+import org.keycloak.sessions.StickySessionEncoderProviderFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+
+public class RemoteStickySessionEncoderProviderFactory implements StickySessionEncoderProviderFactory {
+
+    private static final Logger log = Logger.getLogger(MethodHandles.lookup().lookupClass());
+    private static final char SEPARATOR = '.';
+
+    private static final StickySessionEncoderProvider NO_ROUTER_PROVIDER = new BaseProvider() {
+        @Override
+        public String encodeSessionId(String sessionId) {
+            return sessionId;
+        }
+
+        @Override
+        public boolean shouldAttachRoute() {
+            return false;
+        }
+    };
+
+    private volatile boolean shouldAttachRoute;
+    private volatile StickySessionEncoderProvider provider;
+
+    @Override
+    public StickySessionEncoderProvider create(KeycloakSession session) {
+        return shouldAttachRoute ? provider : NO_ROUTER_PROVIDER;
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+        setShouldAttachRoute(config.getBoolean("shouldAttachRoute", true));
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+        try (var session = factory.create()) {
+            provider = new AttachRouteProvider(getRoute(session));
+        }
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public String getId() {
+        return InfinispanUtils.REMOTE_PROVIDER_ID;
+    }
+
+    @Override
+    public int order() {
+        return InfinispanUtils.PROVIDER_ORDER;
+    }
+
+    @Override
+    public List<ProviderConfigProperty> getConfigMetadata() {
+        return ProviderConfigurationBuilder.create()
+                .property()
+                .name("shouldAttachRoute")
+                .type("boolean")
+                .helpText("If the route should be attached to cookies to reflect the node that owns a particular session.")
+                .defaultValue(true)
+                .add()
+                .build();
+    }
+
+    @Override
+    public boolean isSupported(Config.Scope config) {
+        return InfinispanUtils.isRemoteInfinispan();
+    }
+
+    @Override
+    public void setShouldAttachRoute(boolean shouldAttachRoute) {
+        this.shouldAttachRoute = shouldAttachRoute;
+        log.debugf("Should attach route to the sticky session cookie: %b", shouldAttachRoute);
+    }
+
+    private static String getRoute(KeycloakSession session) {
+        return InfinispanUtil.getTopologyInfo(session).getMyNodeName();
+    }
+
+    private static abstract class BaseProvider implements StickySessionEncoderProvider {
+
+        @Override
+        public final String decodeSessionId(String encodedSessionId) {
+            // Try to decode regardless if shouldAttachRoute is true/false.
+            // It is possible that some loadbalancers may forward the route information attached by them to the backend keycloak server.
+            // We need to remove it then.
+            int index = encodedSessionId.indexOf(SEPARATOR);
+            return index == -1 ? encodedSessionId : encodedSessionId.substring(0, index);
+        }
+
+        @Override
+        public final void close() {
+        }
+    }
+
+    private static class AttachRouteProvider extends BaseProvider {
+
+        private final String route;
+
+        private AttachRouteProvider(String route) {
+            this.route = route;
+        }
+
+        @Override
+        public String encodeSessionId(String sessionId) {
+            return sessionId + SEPARATOR + route;
+        }
+
+        @Override
+        public boolean shouldAttachRoute() {
+            return true;
+        }
+    }
+}

model/infinispan/src/main/resources/META-INF/services/org.keycloak.health.LoadBalancerCheckProviderFactory

@@ -1 +1,2 @@
-org.keycloak.connections.infinispan.InfinispanMultiSiteLoadBalancerCheckProviderFactory
+org.keycloak.connections.infinispan.InfinispanMultiSiteLoadBalancerCheckProviderFactory
+org.keycloak.connections.infinispan.remote.RemoteLoadBalancerCheckProviderFactory

model/infinispan/src/main/resources/META-INF/services/org.keycloak.models.SingleUseObjectProviderFactory

@@ -15,4 +15,5 @@
 # limitations under the License.
 #
 
-org.keycloak.models.sessions.infinispan.InfinispanSingleUseObjectProviderFactory
+org.keycloak.models.sessions.infinispan.InfinispanSingleUseObjectProviderFactory
+org.keycloak.models.sessions.infinispan.remote.RemoteInfinispanSingleUseObjectProviderFactory

model/infinispan/src/main/resources/META-INF/services/org.keycloak.sessions.StickySessionEncoderProviderFactory

@@ -15,4 +15,5 @@
 # limitations under the License.
 #
 
-org.keycloak.models.sessions.infinispan.InfinispanStickySessionEncoderProviderFactory
+org.keycloak.models.sessions.infinispan.InfinispanStickySessionEncoderProviderFactory
+org.keycloak.models.sessions.infinispan.remote.RemoteStickySessionEncoderProviderFactory

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/legacy/infinispan/CacheManagerFactory.java

@@ -19,6 +19,7 @@ package org.keycloak.quarkus.runtime.storage.legacy.infinispan;
 
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -50,6 +51,7 @@ import org.jgroups.util.TLSClientAuth;
 import org.keycloak.common.Profile;
 import org.keycloak.config.CachingOptions;
 import org.keycloak.config.MetricsOptions;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.marshalling.Marshalling;
 import org.keycloak.quarkus.runtime.configuration.Configuration;
 
@@ -63,9 +65,10 @@ import static org.keycloak.config.CachingOptions.CACHE_REMOTE_HOST_PROPERTY;
 import static org.keycloak.config.CachingOptions.CACHE_REMOTE_PASSWORD_PROPERTY;
 import static org.keycloak.config.CachingOptions.CACHE_REMOTE_PORT_PROPERTY;
 import static org.keycloak.config.CachingOptions.CACHE_REMOTE_USERNAME_PROPERTY;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ACTION_TOKEN_CACHE;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME;
-import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.DISTRIBUTED_REPLICATED_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME;
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.USER_SESSION_CACHE_NAME;
@@ -81,7 +84,7 @@ public class CacheManagerFactory {
 
     public CacheManagerFactory(String config) {
         this.cacheManagerFuture = startEmbeddedCacheManager(config);
-        if (isCrossSiteEnabled() && isRemoteCacheEnabled()) {
+        if (InfinispanUtils.isRemoteInfinispan()) {
             logger.debug("Remote Cache feature is enabled");
             this.remoteCacheManagerFuture = CompletableFuture.supplyAsync(this::startRemoteCacheManager);
         } else {
@@ -104,14 +107,6 @@ public class CacheManagerFactory {
         remoteCacheManagerFuture.thenAccept(CacheManagerFactory::close);
     }
 
-    private static boolean isCrossSiteEnabled() {
-        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE);
-    }
-
-    private static boolean isRemoteCacheEnabled() {
-        return Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE);
-    }
-
     private static <T> T join(Future<T> future) {
         try {
             return future.get(getStartTimeout(), TimeUnit.SECONDS);
@@ -162,8 +157,7 @@ public class CacheManagerFactory {
         if (createRemoteCaches()) {
             // fall back for distributed caches if not defined
             logger.warn("Creating remote cache in external Infinispan server. It should not be used in production!");
-            for (String name : DISTRIBUTED_REPLICATED_CACHE_NAMES) {
+            for (String name : CLUSTERED_CACHE_NAMES) {
-
                 builder.remoteCache(name).templateName(DefaultTemplate.DIST_SYNC);
             }
         }
@@ -172,7 +166,7 @@ public class CacheManagerFactory {
 
         // establish connection to all caches
         if (isStartEagerly()) {
-            DISTRIBUTED_REPLICATED_CACHE_NAMES.forEach(remoteCacheManager::getCache);
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(remoteCacheManager::getCache);
         }
         return remoteCacheManager;
     }
@@ -185,7 +179,7 @@ public class CacheManagerFactory {
             configureRemoteStores(builder);
         }
 
-        DISTRIBUTED_REPLICATED_CACHE_NAMES.forEach(cacheName -> {
+        Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(cacheName -> {
             if (cacheName.equals(USER_SESSION_CACHE_NAME) || cacheName.equals(CLIENT_SESSION_CACHE_NAME) || cacheName.equals(OFFLINE_USER_SESSION_CACHE_NAME) || cacheName.equals(OFFLINE_CLIENT_SESSION_CACHE_NAME)) {
                 ConfigurationBuilder configurationBuilder = builder.getNamedConfigurationBuilders().get(cacheName);
                 if (Profile.isFeatureEnabled(Profile.Feature.PERSISTENT_USER_SESSIONS)) {
@@ -222,7 +216,7 @@ public class CacheManagerFactory {
         }
 
         Marshalling.configure(builder.getGlobalConfigurationBuilder());
-        if (isCrossSiteEnabled() && isRemoteCacheEnabled()) {
+        if (InfinispanUtils.isRemoteInfinispan()) {
             var builders = builder.getNamedConfigurationBuilders();
             // remove all distributed caches
             logger.debug("Removing all distributed caches.");
@@ -230,6 +224,7 @@ public class CacheManagerFactory {
             //DISTRIBUTED_REPLICATED_CACHE_NAMES.forEach(builders::remove);
             builders.remove(WORK_CACHE_NAME);
             builders.remove(AUTHENTICATION_SESSIONS_CACHE_NAME);
+            builders.remove(ACTION_TOKEN_CACHE);
         }
 
         var start = isStartEagerly();
@@ -329,7 +324,7 @@ public class CacheManagerFactory {
 
             SSLContext sslContext = createSSLContext();
 
-            DISTRIBUTED_REPLICATED_CACHE_NAMES.forEach(cacheName -> {
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(cacheName -> {
                 PersistenceConfigurationBuilder persistenceCB = builder.getNamedConfigurationBuilders().get(cacheName).persistence();
 
                 //if specified via command line -> cannot be defined in the xml file

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/ExternalInfinispanTest.java

@@ -21,6 +21,7 @@ import io.quarkus.test.junit.main.Launch;
 import io.quarkus.test.junit.main.LaunchResult;
 import org.junit.jupiter.api.Test;
 import org.keycloak.common.util.Retry;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.InfinispanContainer;
@@ -35,10 +36,31 @@ public class ExternalInfinispanTest {
     @Test
     @Launch({ "start-dev", "--features=multi-site", "--cache=ispn", "--cache-config-file=../../../test-classes/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml", "--spi-connections-infinispan-quarkus-site-name=ISPN" })
     void testLoadBalancerCheckFailure() {
+        runLoadBalancerCheckFailureTest();
+    }
+
+    @Test
+    @Launch({
+            "start-dev",
+            "--features=multi-site,remote-cache",
+            "--cache=ispn",
+            "--cache-remote-host=localhost",
+            "--cache-remote-username=keycloak",
+            "--cache-remote-password=Password1!",
+            "--spi-connections-infinispan-quarkus-site-name=ISPN",
+            "--spi-load-balancer-check-remote-poll-interval=500",
+            "-Dkc.cache-remote-tls-enabled=false",
+            "--verbose"
+    })
+    void testLoadBalancerCheckFailureWithRemoteOnlyCaches() {
+        runLoadBalancerCheckFailureTest();
+    }
+
+    private void runLoadBalancerCheckFailureTest() {
         when().get("/lb-check").then()
                 .statusCode(200);
 
-        InfinispanContainer.removeCache("sessions");
+        InfinispanContainer.removeCache(InfinispanConnectionProvider.WORK_CACHE_NAME);
 
         // The `lb-check` relies on the Infinispan's persistence check status. By default, Infinispan checks in the background every second that the remote store is available.
         // So we'll wait on average about one second here for the check to switch its state.

quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/InfinispanContainer.java

@@ -17,18 +17,19 @@
 
 package org.keycloak.it.junit5.extension;
 
+import java.time.Duration;
+import java.util.Arrays;
+
 import org.infinispan.client.hotrod.RemoteCacheManager;
 import org.infinispan.client.hotrod.configuration.ClientIntelligence;
 import org.infinispan.client.hotrod.configuration.ConfigurationBuilder;
-import org.infinispan.commons.configuration.XMLStringConfiguration;
+import org.infinispan.commons.configuration.StringConfiguration;
 import org.jboss.logging.Logger;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.images.PullPolicy;
 
-import java.time.Duration;
-import java.util.stream.Stream;
-
 public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     private final Logger LOG = Logger.getLogger(getClass());
@@ -38,6 +39,7 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     public static RemoteCacheManager remoteCacheManager;
 
+    @SuppressWarnings("resource")
     public InfinispanContainer() {
         super(getImageName());
         withEnv("USER", USERNAME);
@@ -76,7 +78,7 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     private void establishHotRodConnection() {
         ConfigurationBuilder configBuilder = new ConfigurationBuilder()
-                .addServers(getContainerIpAddress() + ":11222")
+                .addServers(getHost() + ":11222")
                 .security()
                 .authentication()
                 .username(getUsername())
@@ -97,7 +99,7 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
         establishHotRodConnection();
 
-        Stream.of("sessions", "actionTokens", "authenticationSessions", "clientSessions", "offlineSessions", "offlineClientSessions", "loginFailures", "work")
+        Arrays.stream(InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES)
                 .forEach(cacheName -> {
                     LOG.infof("Creating cache '%s'", cacheName);
                     createCache(remoteCacheManager, cacheName);
@@ -117,7 +119,7 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     public void createCache(RemoteCacheManager remoteCacheManager, String cacheName) {
         String xml = String.format("<distributed-cache name=\"%s\" mode=\"SYNC\" owners=\"2\"></distributed-cache>" , cacheName);
-        remoteCacheManager.administration().getOrCreateCache(cacheName, new XMLStringConfiguration(xml));
+        remoteCacheManager.administration().getOrCreateCache(cacheName, new StringConfiguration(xml));
     }
 
     public String getPort() {

server-spi-private/src/main/java/org/keycloak/health/LoadBalancerCheckProvider.java

@@ -24,6 +24,7 @@ import org.keycloak.provider.Provider;
  * the load balancer endpoint will return the {@code DOWN} status.
  *
  */
+@FunctionalInterface
 public interface LoadBalancerCheckProvider extends Provider {
 
     /**
@@ -39,4 +40,9 @@ public interface LoadBalancerCheckProvider extends Provider {
      * @return true if the component is down/unhealthy, false otherwise
      */
     boolean isDown();
+
+    @Override
+    default void close() {
+        //no-op by default
+    }
 }

server-spi-private/src/main/java/org/keycloak/models/SingleUseObjectProviderFactory.java

@@ -17,10 +17,11 @@
 
 package org.keycloak.models;
 
+import org.keycloak.provider.EnvironmentDependentProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public interface SingleUseObjectProviderFactory<T extends SingleUseObjectProvider> extends ProviderFactory<T> {
+public interface SingleUseObjectProviderFactory<T extends SingleUseObjectProvider> extends ProviderFactory<T>, EnvironmentDependentProviderFactory {
 }

server-spi-private/src/main/java/org/keycloak/sessions/StickySessionEncoderProviderFactory.java

@@ -17,10 +17,17 @@
 
 package org.keycloak.sessions;
 
+import org.keycloak.provider.EnvironmentDependentProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public interface StickySessionEncoderProviderFactory extends ProviderFactory<StickySessionEncoderProvider> {
+public interface StickySessionEncoderProviderFactory extends ProviderFactory<StickySessionEncoderProvider>, EnvironmentDependentProviderFactory {
+
+    /**
+     * For testing purpose only
+     */
+    void setShouldAttachRoute(boolean shouldAttachRoute);
+
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AuthenticationSessionClusterTest.java

@@ -25,10 +25,10 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
-import org.keycloak.models.sessions.infinispan.InfinispanStickySessionEncoderProviderFactory;
 import org.keycloak.connections.infinispan.InfinispanUtil;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.sessions.StickySessionEncoderProvider;
+import org.keycloak.sessions.StickySessionEncoderProviderFactory;
 import org.keycloak.testsuite.pages.AppPage;
 import org.keycloak.testsuite.pages.LoginPage;
 import org.keycloak.testsuite.pages.LoginPasswordUpdatePage;
@@ -117,7 +117,7 @@ public class AuthenticationSessionClusterTest extends AbstractClusterTest {
 
         // Disable route on backend server
         getTestingClientFor(backendNode(0)).server().run(session -> {
-            InfinispanStickySessionEncoderProviderFactory factory = (InfinispanStickySessionEncoderProviderFactory) session.getKeycloakSessionFactory().getProviderFactory(StickySessionEncoderProvider.class, "infinispan");
+            StickySessionEncoderProviderFactory factory = (StickySessionEncoderProviderFactory) session.getKeycloakSessionFactory().getProviderFactory(StickySessionEncoderProvider.class);
             factory.setShouldAttachRoute(false);
         });
 
@@ -141,7 +141,7 @@ public class AuthenticationSessionClusterTest extends AbstractClusterTest {
 
         // Revert route on backend server
         getTestingClientFor(backendNode(0)).server().run(session -> {
-            InfinispanStickySessionEncoderProviderFactory factory = (InfinispanStickySessionEncoderProviderFactory) session.getKeycloakSessionFactory().getProviderFactory(StickySessionEncoderProvider.class, "infinispan");
+            StickySessionEncoderProviderFactory factory = (StickySessionEncoderProviderFactory) session.getKeycloakSessionFactory().getProviderFactory(StickySessionEncoderProvider.class);
             factory.setShouldAttachRoute(true);
         });
     }

testsuite/model/pom.xml

@@ -238,6 +238,19 @@
             <properties>
                 <keycloak.model.parameters>CrossDCInfinispan,Jpa</keycloak.model.parameters>
             </properties>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <configuration>
+                            <systemPropertyVariables>
+                                <keycloak.profile.feature.multi_site>enabled</keycloak.profile.feature.multi_site>
+                            </systemPropertyVariables>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
         </profile>
 
         <profile>

testsuite/model/src/test/java/org/keycloak/testsuite/model/infinispan/FeatureEnabledTest.java

@@ -0,0 +1,114 @@
+package org.keycloak.testsuite.model.infinispan;
+
+import java.util.Arrays;
+import java.util.function.Predicate;
+
+import org.infinispan.commons.CacheConfigurationException;
+import org.junit.Test;
+import org.keycloak.common.Profile;
+import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
+import org.keycloak.infinispan.util.InfinispanUtils;
+import org.keycloak.testsuite.model.KeycloakModelTest;
+import org.keycloak.testsuite.model.RequireProvider;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ACTION_TOKEN_CACHE;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_CACHE_NAMES;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.WORK_CACHE_NAME;
+
+/**
+ * Checks if the correct embedded or remote cache is started based on {@link org.keycloak.common.Profile.Feature}.
+ */
+@RequireProvider(InfinispanConnectionProvider.class)
+public class FeatureEnabledTest extends KeycloakModelTest {
+
+    @Test
+    public void testLocalCaches() {
+        inComittedTransaction(session -> {
+            var clusterProvider = session.getProvider(InfinispanConnectionProvider.class);
+            for (var cacheName : LOCAL_CACHE_NAMES) {
+                assertEmbeddedCacheExists(clusterProvider, cacheName);
+            }
+        });
+    }
+
+    @Test
+    public void testRemoteCachesOnly() {
+        assumeTrue("Remote-Cache Feature disabled", Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE));
+        assumeTrue("Multi-Site Feature disabled", Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE));
+        assertTrue(InfinispanUtils.isRemoteInfinispan());
+        assertFalse(InfinispanUtils.isEmbeddedInfinispan());
+        inComittedTransaction(session -> {
+            var clusterProvider = session.getProvider(InfinispanConnectionProvider.class);
+            assertEmbeddedCacheDoesNotExists(clusterProvider, WORK_CACHE_NAME);
+            assertEmbeddedCacheDoesNotExists(clusterProvider, AUTHENTICATION_SESSIONS_CACHE_NAME);
+            assertEmbeddedCacheDoesNotExists(clusterProvider, ACTION_TOKEN_CACHE);
+
+            // TODO [pruivo] all caches eventually won't exists in embedded
+            Arrays.stream(CLUSTERED_CACHE_NAMES)
+                    .filter(Predicate.not(Predicate.isEqual(WORK_CACHE_NAME)))
+                    .filter(Predicate.not(Predicate.isEqual(AUTHENTICATION_SESSIONS_CACHE_NAME)))
+                    .filter(Predicate.not(Predicate.isEqual(ACTION_TOKEN_CACHE)))
+                    .forEach(s -> assertEmbeddedCacheExists(clusterProvider, s));
+
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(s -> assertRemoteCacheExists(clusterProvider, s));
+
+        });
+    }
+
+    @Test
+    public void testRemoteAndEmbeddedCaches() {
+        assumeTrue("Multi-Site Feature disabled", Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE));
+        assumeFalse("Remote-Cache Feature enabled", Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE));
+        assertFalse(InfinispanUtils.isRemoteInfinispan());
+        assertTrue(InfinispanUtils.isEmbeddedInfinispan());
+        inComittedTransaction(session -> {
+            var clusterProvider = session.getProvider(InfinispanConnectionProvider.class);
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(s -> assertEmbeddedCacheExists(clusterProvider, s));
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(s -> assertRemoteCacheExists(clusterProvider, s));
+        });
+    }
+
+    @Test
+    public void testEmbeddedCachesOnly() {
+        assumeFalse("Multi-Site Feature enabled", Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE));
+        assumeFalse("Remote-Cache Feature enabled", Profile.isFeatureEnabled(Profile.Feature.REMOTE_CACHE));
+        assertFalse(InfinispanUtils.isRemoteInfinispan());
+        assertTrue(InfinispanUtils.isEmbeddedInfinispan());
+        inComittedTransaction(session -> {
+            var clusterProvider = session.getProvider(InfinispanConnectionProvider.class);
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(s -> assertEmbeddedCacheExists(clusterProvider, s));
+            Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(s -> assertRemoteCacheDoesNotExists(clusterProvider, s));
+        });
+    }
+
+    private static void assertEmbeddedCacheExists(InfinispanConnectionProvider provider, String cacheName) {
+        assertNotNull(String.format("Embedded cache '%s' should exist", cacheName), provider.getCache(cacheName));
+    }
+
+    private static void assertEmbeddedCacheDoesNotExists(InfinispanConnectionProvider provider, String cacheName) {
+        try {
+            provider.getCache(cacheName);
+            fail(String.format("Embedded cache '%s' should not exist", cacheName));
+        } catch (CacheConfigurationException expected) {
+            // expected
+        }
+    }
+
+    private static void assertRemoteCacheExists(InfinispanConnectionProvider provider, String cacheName) {
+        assertNotNull(String.format("Remote cache '%s' should exist", cacheName), provider.getRemoteCache(cacheName));
+    }
+
+    private static void assertRemoteCacheDoesNotExists(InfinispanConnectionProvider provider, String cacheName) {
+        assertNull(String.format("Remote cache '%s' should not exist", cacheName), provider.getRemoteCache(cacheName));
+    }
+
+}

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/CrossDCInfinispan.java

@@ -18,11 +18,11 @@ package org.keycloak.testsuite.model.parameters;
 
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.UserSessionSpi;
-import org.keycloak.models.sessions.infinispan.InfinispanUserSessionProviderFactory;
 import org.keycloak.testsuite.model.Config;
-import org.keycloak.testsuite.model.KeycloakModelParameters;
 import org.keycloak.testsuite.model.HotRodServerRule;
+import org.keycloak.testsuite.model.KeycloakModelParameters;
 
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
@@ -59,7 +59,7 @@ public class CrossDCInfinispan extends KeycloakModelParameters {
                     .config("jgroupsUdpMcastAddr", mcastAddr(NODE_COUNTER.get()))
                     .config("jgroupsBindAddr", "127.0.0.1") // bind to localhost for testing
                     .spi(UserSessionSpi.NAME)
-                    .provider(InfinispanUserSessionProviderFactory.PROVIDER_ID)
+                    .provider(InfinispanUtils.EMBEDDED_PROVIDER_ID)
                     .config("offlineSessionCacheEntryLifespanOverride", "43200")
                     .config("offlineClientSessionCacheEntryLifespanOverride", "43200");
         }

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/Infinispan.java

@@ -16,35 +16,36 @@
  */
 package org.keycloak.testsuite.model.parameters;
 
+import com.google.common.collect.ImmutableSet;
 import org.keycloak.cluster.infinispan.InfinispanClusterProviderFactory;
 import org.keycloak.connections.infinispan.InfinispanConnectionProviderFactory;
 import org.keycloak.connections.infinispan.InfinispanConnectionSpi;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.keys.PublicKeyStorageSpi;
 import org.keycloak.keys.infinispan.InfinispanCachePublicKeyProviderFactory;
 import org.keycloak.keys.infinispan.InfinispanPublicKeyStorageProviderFactory;
 import org.keycloak.models.SingleUseObjectSpi;
 import org.keycloak.models.UserLoginFailureSpi;
 import org.keycloak.models.UserSessionSpi;
-import org.keycloak.models.cache.authorization.CachedStoreFactorySpi;
-import org.keycloak.models.cache.infinispan.authorization.InfinispanCacheStoreFactoryProviderFactory;
 import org.keycloak.models.cache.CachePublicKeyProviderSpi;
+import org.keycloak.models.cache.CacheRealmProviderSpi;
+import org.keycloak.models.cache.CacheUserProviderSpi;
+import org.keycloak.models.cache.authorization.CachedStoreFactorySpi;
+import org.keycloak.models.cache.infinispan.InfinispanCacheRealmProviderFactory;
+import org.keycloak.models.cache.infinispan.InfinispanUserCacheProviderFactory;
+import org.keycloak.models.cache.infinispan.authorization.InfinispanCacheStoreFactoryProviderFactory;
 import org.keycloak.models.session.UserSessionPersisterSpi;
 import org.keycloak.models.sessions.infinispan.InfinispanAuthenticationSessionProviderFactory;
 import org.keycloak.models.sessions.infinispan.InfinispanSingleUseObjectProviderFactory;
 import org.keycloak.models.sessions.infinispan.InfinispanUserLoginFailureProviderFactory;
 import org.keycloak.models.sessions.infinispan.InfinispanUserSessionProviderFactory;
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
 import org.keycloak.sessions.AuthenticationSessionSpi;
 import org.keycloak.sessions.StickySessionEncoderProviderFactory;
 import org.keycloak.sessions.StickySessionEncoderSpi;
-import org.keycloak.testsuite.model.KeycloakModelParameters;
-import org.keycloak.models.cache.CacheRealmProviderSpi;
-import org.keycloak.models.cache.CacheUserProviderSpi;
-import org.keycloak.models.cache.infinispan.InfinispanCacheRealmProviderFactory;
-import org.keycloak.models.cache.infinispan.InfinispanUserCacheProviderFactory;
-import org.keycloak.provider.ProviderFactory;
-import org.keycloak.provider.Spi;
 import org.keycloak.testsuite.model.Config;
-import com.google.common.collect.ImmutableSet;
+import org.keycloak.testsuite.model.KeycloakModelParameters;
 import org.keycloak.timer.TimerProviderFactory;
 
 import java.util.Set;
@@ -97,10 +98,10 @@ public class Infinispan extends KeycloakModelParameters {
                 .config("useKeycloakTimeService", "true")
                 .config("nodeName", "node-" + NODE_COUNTER.incrementAndGet())
                 .spi(UserLoginFailureSpi.NAME)
-                .provider(InfinispanUserLoginFailureProviderFactory.PROVIDER_ID)
+                .provider(InfinispanUtils.EMBEDDED_PROVIDER_ID)
                 .config("stalledTimeoutInSeconds", "10")
                 .spi(UserSessionSpi.NAME)
-                .provider(InfinispanUserSessionProviderFactory.PROVIDER_ID)
+                .provider(InfinispanUtils.EMBEDDED_PROVIDER_ID)
                 .config("sessionPreloadStalledTimeoutInSeconds", "10")
                 .config("offlineSessionCacheEntryLifespanOverride", "43200")
                 .config("offlineClientSessionCacheEntryLifespanOverride", "43200")

testsuite/model/src/test/java/org/keycloak/testsuite/model/parameters/RemoteInfinispan.java

@@ -20,9 +20,12 @@ import com.google.common.collect.ImmutableSet;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 import org.keycloak.cluster.infinispan.remote.RemoteInfinispanClusterProviderFactory;
+import org.keycloak.connections.infinispan.remote.RemoteLoadBalancerCheckProviderFactory;
+import org.keycloak.infinispan.util.InfinispanUtils;
 import org.keycloak.models.UserSessionSpi;
-import org.keycloak.models.sessions.infinispan.InfinispanUserSessionProviderFactory;
 import org.keycloak.models.sessions.infinispan.remote.RemoteInfinispanAuthenticationSessionProviderFactory;
+import org.keycloak.models.sessions.infinispan.remote.RemoteInfinispanSingleUseObjectProviderFactory;
+import org.keycloak.models.sessions.infinispan.remote.RemoteStickySessionEncoderProviderFactory;
 import org.keycloak.provider.ProviderFactory;
 import org.keycloak.testsuite.model.Config;
 import org.keycloak.testsuite.model.HotRodServerRule;
@@ -53,6 +56,9 @@ public class RemoteInfinispan extends KeycloakModelParameters {
             .addAll(Infinispan.ALLOWED_FACTORIES)
             .add(RemoteInfinispanClusterProviderFactory.class)
             .add(RemoteInfinispanAuthenticationSessionProviderFactory.class)
+            .add(RemoteInfinispanSingleUseObjectProviderFactory.class)
+            .add(RemoteStickySessionEncoderProviderFactory.class)
+            .add(RemoteLoadBalancerCheckProviderFactory.class)
             .build();
 
     @Override
@@ -71,7 +77,7 @@ public class RemoteInfinispan extends KeycloakModelParameters {
                     .config("remoteStorePort", siteName(NODE_COUNTER.get()).equals("site-2") ? "11333" : "11222")
                     .config("jgroupsUdpMcastAddr", mcastAddr(NODE_COUNTER.get()))
                     .spi(UserSessionSpi.NAME)
-                    .provider(InfinispanUserSessionProviderFactory.PROVIDER_ID)
+                    .provider(InfinispanUtils.EMBEDDED_PROVIDER_ID)
                     .config("offlineSessionCacheEntryLifespanOverride", "43200")
                     .config("offlineClientSessionCacheEntryLifespanOverride", "43200");
         }

testsuite/model/src/test/java/org/keycloak/testsuite/model/session/SessionTimeoutsTest.java

@@ -93,14 +93,14 @@ public class SessionTimeoutsTest extends KeycloakModelTest {
     private void clearSessionCaches(KeycloakSession s) {
         InfinispanConnectionProvider provider = s.getProvider(InfinispanConnectionProvider.class);
         if (provider != null) {
-            for (String cache : InfinispanConnectionProvider.DISTRIBUTED_REPLICATED_CACHE_NAMES) {
+            for (String cache : InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES) {
                 provider.getCache(cache).clear();
             }
         }
 
         HotRodServerRule hotRodServer = getParameters(HotRodServerRule.class).findFirst().orElse(null);
         if (hotRodServer != null) {
-           for (String cache : InfinispanConnectionProvider.DISTRIBUTED_REPLICATED_CACHE_NAMES) {
+           for (String cache : InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES) {
                hotRodServer.getHotRodCacheManager().getCache(cache).clear();
                hotRodServer.getHotRodCacheManager2().getCache(cache).clear();
            }