Publish information about Infinispan availability in lb-check if MULTI_SITE is enabled

Closes #25077

Signed-off-by: Michal Hajas <mhajas@redhat.com>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>
Co-authored-by: Pedro Ruivo <pruivo@redhat.com>
Co-authored-by: Alexander Schwartz <aschwart@redhat.com>

common/src/main/java/org/keycloak/common/Profile.java

@@ -86,7 +86,7 @@ public class Profile {
 
         UPDATE_EMAIL("Update Email Action", Type.PREVIEW),
 
-        JS_ADAPTER("Host keycloak.js and keycloak-authz.js through the Keycloak sever", Type.DEFAULT),
+        JS_ADAPTER("Host keycloak.js and keycloak-authz.js through the Keycloak server", Type.DEFAULT),
 
         FIPS("FIPS 140-2 mode", Type.DISABLED_BY_DEFAULT),
 
@@ -97,6 +97,8 @@ public class Profile {
         DEVICE_FLOW("OAuth 2.0 Device Authorization Grant", Type.DEFAULT),
 
         TRANSIENT_USERS("Transient users for brokering", Type.EXPERIMENTAL),
+
+        MULTI_SITE("Multi-site support", Type.PREVIEW),
         ;
 
         private final Type type;

common/src/test/java/org/keycloak/common/ProfileTest.java

@@ -78,6 +78,7 @@ public class ProfileTest {
             Profile.Feature.ADMIN_FINE_GRAINED_AUTHZ,
             Profile.Feature.DYNAMIC_SCOPES,
             Profile.Feature.DOCKER,
+            Profile.Feature.MULTI_SITE,
             Profile.Feature.RECOVERY_CODES,
             Profile.Feature.SCRIPTS,
             Profile.Feature.TOKEN_EXCHANGE,
@@ -93,7 +94,7 @@ public class ProfileTest {
             disabledFeatures.add(Profile.Feature.KERBEROS);
         }
         assertEquals(profile.getDisabledFeatures(), disabledFeatures);
-        assertEquals(profile.getPreviewFeatures(), Profile.Feature.ACCOUNT3, Profile.Feature.ADMIN_FINE_GRAINED_AUTHZ, Profile.Feature.RECOVERY_CODES, Profile.Feature.SCRIPTS, Profile.Feature.TOKEN_EXCHANGE, Profile.Feature.DECLARATIVE_USER_PROFILE, Profile.Feature.CLIENT_SECRET_ROTATION, Profile.Feature.UPDATE_EMAIL, Profile.Feature.DPOP);
+        assertEquals(profile.getPreviewFeatures(), Profile.Feature.ACCOUNT3, Profile.Feature.ADMIN_FINE_GRAINED_AUTHZ, Profile.Feature.MULTI_SITE, Profile.Feature.RECOVERY_CODES, Profile.Feature.SCRIPTS, Profile.Feature.TOKEN_EXCHANGE, Profile.Feature.DECLARATIVE_USER_PROFILE, Profile.Feature.CLIENT_SECRET_ROTATION, Profile.Feature.UPDATE_EMAIL, Profile.Feature.DPOP);
     }
 
     @Test

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProvider.java

@@ -37,8 +37,8 @@ public class DefaultInfinispanConnectionProvider implements InfinispanConnection
     }
 
     @Override
-    public <K, V> Cache<K, V> getCache(String name) {
-        return cacheManager.getCache(name);
+    public <K, V> Cache<K, V> getCache(String name, boolean createIfAbsent) {
+        return cacheManager.getCache(name, createIfAbsent);
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -17,6 +17,7 @@
 
 package org.keycloak.connections.infinispan;
 
+import org.infinispan.Cache;
 import org.infinispan.client.hotrod.ProtocolVersion;
 import org.infinispan.commons.dataconversion.MediaType;
 import org.infinispan.configuration.cache.CacheMode;
@@ -28,6 +29,7 @@ import org.infinispan.eviction.EvictionType;
 import org.infinispan.jboss.marshalling.core.JBossUserMarshaller;
 import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
+import org.infinispan.persistence.manager.PersistenceManager;
 import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
 import org.infinispan.transaction.LockingMode;
 import org.infinispan.transaction.TransactionMode;
@@ -119,7 +121,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
     public void postInit(KeycloakSessionFactory factory) {
         factory.register((ProviderEvent event) -> {
             if (event instanceof PostMigrationEvent) {
-                KeycloakModelUtils.runJobInTransaction(factory, session -> { registerSystemWideListeners(session); });
+                KeycloakModelUtils.runJobInTransaction(factory, this::registerSystemWideListeners);
             }
         });
     }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -66,7 +66,43 @@ public interface InfinispanConnectionProvider extends Provider {
     // Constant used as the prefix of the current node if "jboss.node.name" is not configured
     String NODE_PREFIX = "node_";
 
-    <K, V> Cache<K, V> getCache(String name);
+    String[] ALL_CACHES_NAME = {
+            REALM_CACHE_NAME,
+            REALM_REVISIONS_CACHE_NAME,
+            USER_CACHE_NAME,
+            USER_REVISIONS_CACHE_NAME,
+            USER_SESSION_CACHE_NAME,
+            CLIENT_SESSION_CACHE_NAME,
+            OFFLINE_USER_SESSION_CACHE_NAME,
+            OFFLINE_CLIENT_SESSION_CACHE_NAME,
+            LOGIN_FAILURE_CACHE_NAME,
+            AUTHENTICATION_SESSIONS_CACHE_NAME,
+            WORK_CACHE_NAME,
+            AUTHORIZATION_CACHE_NAME,
+            AUTHORIZATION_REVISIONS_CACHE_NAME,
+            ACTION_TOKEN_CACHE,
+            KEYS_CACHE_NAME
+    };
+
+    /**
+     *
+     * Effectively the same as {@link InfinispanConnectionProvider#getCache(String, boolean)} with createIfAbsent set to {@code true}
+     *
+     */
+    default <K, V> Cache<K, V> getCache(String name) {
+        return getCache(name, true);
+    }
+
+    /**
+     * Provides an instance if Infinispan cache by name
+     *
+     * @param name name of the requested cache
+     * @param createIfAbsent if true the connection provider will create the requested cache on method call if it does not exist
+     * @return return a cache instance
+     * @param <K> key type
+     * @param <V> value type
+     */
+    <K, V> Cache<K, V> getCache(String name, boolean createIfAbsent);
 
     /**
      * Get remote cache of given name. Could just retrieve the remote cache from the remoteStore configured in given infinispan cache and/or

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanMultiSiteLoadBalancerCheckProvider.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.connections.infinispan;
+
+import org.infinispan.Cache;
+import org.infinispan.persistence.manager.PersistenceManager;
+import org.jboss.logging.Logger;
+import org.keycloak.health.LoadBalancerCheckProvider;
+
+import java.util.Objects;
+
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ALL_CACHES_NAME;
+
+public class InfinispanMultiSiteLoadBalancerCheckProvider implements LoadBalancerCheckProvider {
+    private static final Logger LOG = Logger.getLogger(InfinispanMultiSiteLoadBalancerCheckProvider.class);
+    private final InfinispanConnectionProvider connectionProvider;
+
+    public InfinispanMultiSiteLoadBalancerCheckProvider(InfinispanConnectionProvider connectionProvider) {
+        Objects.requireNonNull(connectionProvider, "connectionProvider");
+        this.connectionProvider = connectionProvider;
+    }
+
+    /**
+     * Non-blocking check if all caches and their persistence are available.
+     * <p />
+     * In a situation where any cache's remote cache is unreachable, this will report the "down" to the caller.
+     * When the remote cache is down, it assumes that it is down for all Keycloak nodes in this site, all incoming
+     * requests are likely to fail and that a loadbalancer should send traffic to the other site that might be healthy.
+     * <p />
+     * This code is non-blocking as the embedded Infinispan checks the connection to the remote store periodically
+     * in the background (default: every second).
+     * See {@link LoadBalancerCheckProvider#isDown()} to read more why this needs to be non-blocking.
+     *
+     * @return true if the component is down/unhealthy, false otherwise
+     */
+    @Override
+    public boolean isDown() {
+        for (String cacheName : ALL_CACHES_NAME) {
+            // do not block in cache creation, as this method is required to be non-blocking
+            Cache<?,?> cache = connectionProvider.getCache(cacheName, false);
+
+            // check if cache is started
+            if (cache == null || !cache.getStatus().allowInvocations()) {
+                LOG.debugf("Cache '%s' is not started yet.", cacheName);
+                return true; // no need to check other caches
+            }
+
+            PersistenceManager persistenceManager = cache.getAdvancedCache()
+                    .getComponentRegistry()
+                    .getComponent(PersistenceManager.class);
+
+            if (persistenceManager != null && !persistenceManager.isAvailable()) {
+                LOG.debugf("PersistenceManager for cache '%s' is down.", cacheName);
+                return true; // no need to check other caches
+            }
+            LOG.debugf("Cache '%s' is up.", cacheName);
+        }
+
+        return false;
+    }
+
+    @Override
+    public void close() {
+
+    }
+}

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanMultiSiteLoadBalancerCheckProviderFactory.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.connections.infinispan;
+
+import org.jboss.logging.Logger;
+import org.keycloak.Config;
+import org.keycloak.common.Profile;
+import org.keycloak.health.LoadBalancerCheckProvider;
+import org.keycloak.health.LoadBalancerCheckProviderFactory;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.models.KeycloakSessionFactory;
+import org.keycloak.provider.EnvironmentDependentProviderFactory;
+
+
+
+public class InfinispanMultiSiteLoadBalancerCheckProviderFactory implements LoadBalancerCheckProviderFactory, EnvironmentDependentProviderFactory {
+
+    private LoadBalancerCheckProvider loadBalancerCheckProvider;
+    private static final LoadBalancerCheckProvider ALWAYS_HEALTHY = new LoadBalancerCheckProvider() {
+        @Override public boolean isDown() { return false; }
+        @Override public void close() {}
+    };
+    private static final Logger LOG = Logger.getLogger(InfinispanMultiSiteLoadBalancerCheckProviderFactory.class);
+
+    @Override
+    public LoadBalancerCheckProvider create(KeycloakSession session) {
+        if (loadBalancerCheckProvider == null) {
+            InfinispanConnectionProvider infinispanConnectionProvider = session.getProvider(InfinispanConnectionProvider.class);
+            if (infinispanConnectionProvider == null) {
+                LOG.warn("InfinispanConnectionProvider is not available. Load balancer check will be always healthy for Infinispan.");
+                loadBalancerCheckProvider = ALWAYS_HEALTHY;
+            } else {
+                loadBalancerCheckProvider = new InfinispanMultiSiteLoadBalancerCheckProvider(infinispanConnectionProvider);
+            }
+        }
+        return loadBalancerCheckProvider;
+    }
+
+    @Override
+    public void init(Config.Scope config) {
+
+    }
+
+    @Override
+    public void postInit(KeycloakSessionFactory factory) {
+
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    @Override
+    public String getId() {
+        return "infinispan-multisite";
+    }
+
+    @Override
+    public boolean isSupported() {
+        return Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE);
+    }
+}

model/infinispan/src/main/resources/META-INF/services/org.keycloak.health.LoadBalancerCheckProviderFactory

@@ -0,0 +1 @@
+org.keycloak.connections.infinispan.InfinispanMultiSiteLoadBalancerCheckProviderFactory

quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/FeaturesDistTest.java

@@ -28,7 +28,7 @@ import static org.keycloak.quarkus.runtime.cli.command.AbstractStartCommand.OPTI
 @LegacyStore
 public class FeaturesDistTest {
 
-    private static final String PREVIEW_FEATURES_EXPECTED_LOG = "Preview features enabled: account3, admin-fine-grained-authz, client-secret-rotation, declarative-user-profile, dpop, recovery-codes, scripts, token-exchange, update-email";
+    private static final String PREVIEW_FEATURES_EXPECTED_LOG = "Preview features enabled: account3, admin-fine-grained-authz, client-secret-rotation, declarative-user-profile, dpop, multi-site, recovery-codes, scripts, token-exchange, update-email";
 
     @Test
     public void testEnableOnBuild(KeycloakDistribution dist) {

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testBuildHelp.unix.approved.txt

@@ -48,17 +48,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 HTTP/TLS:
 

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testExportHelp.unix.approved.txt

@@ -59,17 +59,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Config:
 
@@ -142,4 +142,4 @@ Export:
 --users-per-file <number>
                      Set the number of users per file. It is used only if 'users' is set to
                        'different_files'. Increasing this number leads to exponentially increasing
-                       export times. Default: 50.
+                       export times. Default: 50.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testExportHelpAll.unix.approved.txt

@@ -59,17 +59,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Config:
 
@@ -142,4 +142,4 @@ Export:
 --users-per-file <number>
                      Set the number of users per file. It is used only if 'users' is set to
                        'different_files'. Increasing this number leads to exponentially increasing
-                       export times. Default: 50.
+                       export times. Default: 50.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testImportHelp.unix.approved.txt

@@ -59,17 +59,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Config:
 
@@ -136,4 +136,4 @@ Import:
 --file <file>        Set the path to a file that will be read.
 --override <true|false>
                      Set if existing data should be overwritten. If set to false, data will be
-                       ignored. Default: true.
+                       ignored. Default: true.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testImportHelpAll.unix.approved.txt

@@ -59,17 +59,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Config:
 
@@ -136,4 +136,4 @@ Import:
 --file <file>        Set the path to a file that will be read.
 --override <true|false>
                      Set if existing data should be overwritten. If set to false, data will be
-                       ignored. Default: true.
+                       ignored. Default: true.

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelp.unix.approved.txt

@@ -75,17 +75,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Hostname:
 

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartDevHelpAll.unix.approved.txt

@@ -75,17 +75,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Hostname:
 

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelp.unix.approved.txt

@@ -76,17 +76,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Hostname:
 

quarkus/tests/integration/src/test/resources/org/keycloak/it/cli/dist/approvals/cli/help/HelpCommandDistTest.testStartHelpAll.unix.approved.txt

@@ -76,17 +76,17 @@ Feature:
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 --features-disabled <feature>
                      Disables a set of one or more features. Possible values are: account-api,
                        account2, account3, admin-api, admin-fine-grained-authz, admin2,
                        authorization, ciba, client-policies, client-secret-rotation,
                        declarative-user-profile, device-flow, docker, dpop, dynamic-scopes, fips,
-                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage, par,
-                       preview, recovery-codes, scripts, step-up-authentication, token-exchange,
-                       transient-users, update-email, web-authn.
+                       impersonation, js-adapter, kerberos, linkedin-oauth, map-storage,
+                       multi-site, par, preview, recovery-codes, scripts, step-up-authentication,
+                       token-exchange, transient-users, update-email, web-authn.
 
 Hostname:
 

server-spi-private/src/main/java/org/keycloak/health/LoadBalancerCheckProvider.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.health;
+
+import org.keycloak.provider.Provider;
+
+/**
+ * This interface is used for controlling load balancer. If one of the implementations reports that it is down,
+ * the load balancer endpoint will return the {@code DOWN} status.
+ *
+ */
+public interface LoadBalancerCheckProvider extends Provider {
+
+    /**
+     * Check if a component represented by this check is down/unhealthy.
+     * <p />
+     * The implementation must be non-blocking as it is executed in the event loop.
+     * It is necessary to run this in the event loop as blocking requests are queued and then the check
+     * would time out on the loadbalancer side when there is an overload situation in Keycloak.
+     * An automatic failover to the secondary site due to an overloaded primary site is desired as this could
+     * lead to a ping-pong between the sites where the primary site becomes available again once the switchover
+     * is complete.
+     *
+     * @return true if the component is down/unhealthy, false otherwise
+     */
+    boolean isDown();
+}

server-spi-private/src/main/java/org/keycloak/health/LoadBalancerCheckProviderFactory.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.health;
+
+import org.keycloak.provider.ProviderFactory;
+
+public interface LoadBalancerCheckProviderFactory extends ProviderFactory<LoadBalancerCheckProvider> {
+}

server-spi-private/src/main/java/org/keycloak/health/LoadBalancerCheckSpi.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2023 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.health;
+
+import org.keycloak.provider.Provider;
+import org.keycloak.provider.ProviderFactory;
+import org.keycloak.provider.Spi;
+
+public class LoadBalancerCheckSpi implements Spi {
+    @Override
+    public boolean isInternal() {
+        return true;
+    }
+
+    @Override
+    public String getName() {
+        return "load-balancer-check";
+    }
+
+    @Override
+    public Class<? extends Provider> getProviderClass() {
+        return LoadBalancerCheckProvider.class;
+    }
+
+    @Override
+    public Class<? extends ProviderFactory> getProviderFactoryClass() {
+        return LoadBalancerCheckProviderFactory.class;
+    }
+}

server-spi-private/src/main/resources/META-INF/services/org.keycloak.provider.Spi

@@ -91,3 +91,4 @@ org.keycloak.services.clientpolicy.executor.ClientPolicyExecutorSpi
 org.keycloak.services.clientpolicy.ClientPolicyManagerSpi
 org.keycloak.userprofile.UserProfileSpi
 org.keycloak.device.DeviceRepresentationSpi
+org.keycloak.health.LoadBalancerCheckSpi

services/pom.xml

@@ -201,6 +201,10 @@
             <groupId>org.eclipse.microprofile.openapi</groupId>
             <artifactId>microprofile-openapi-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.smallrye.common</groupId>
+            <artifactId>smallrye-common-annotation</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -110,6 +110,12 @@ public class KeycloakApplication extends Application {
             singletons.add(new ObjectMapperResolver());
             classes.add(WelcomeResource.class);
 
+            if (Profile.isFeatureEnabled(Profile.Feature.MULTI_SITE)) {
+                // If we are running in multi-site mode, we need to add a resource which to expose
+                // an endpoint for the load balancer to gather information whether this site should receive requests or not.
+                classes.add(LoadBalancerResource.class);
+            }
+
             platform.onStartup(this::startup);
             platform.onShutdown(this::shutdown);
 

services/src/main/java/org/keycloak/services/resources/LoadBalancerResource.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.services.resources;
+
+import io.smallrye.common.annotation.NonBlocking;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.Response;
+import org.jboss.logging.Logger;
+import org.keycloak.health.LoadBalancerCheckProvider;
+import org.keycloak.models.KeycloakSession;
+import org.keycloak.utils.MediaType;
+
+import java.util.Set;
+
+/**
+ * Prepare information for the load balancer (possibly in a multi-site setup) whether this Keycloak cluster should receive traffic.
+ * <p>
+ * This is non-blocking, so that the load balancer can still retrieve the status even if the Keycloak instance is
+ * trying to withstand a high load. See {@link LoadBalancerCheckProvider#isDown()} for a longer explanation.
+ *
+ * @author <a href="mailto:aschwart@redhat.com">Alexander Schwartz</a>
+ */
+@Path("/lb-check")
+@NonBlocking
+public class LoadBalancerResource {
+
+    protected static final Logger logger = Logger.getLogger(LoadBalancerResource.class);
+
+    @Context
+    KeycloakSession session;
+
+    /**
+     * Return the status for a laod balancer in a multi-site setup if this Keycloak site should receive traffic.
+     * <p />
+     * While a loadbalancer will usually check for the returned status code, the additional text <code>UP</code> or <code>DOWN</down>
+     * is returned for humans to see the status in the browser.
+     * <p />
+     * In contrast to other management endpoints of Quarkus, no information is returned to the caller about the internal state of Keycloak
+     * as this endpoint might be publicly available from the internet and should return as little information as possible.
+     *
+     * @return HTTP status 503 and DOWN when down, and HTTP status 200 and UP when up.
+     */
+    @GET
+    @Produces(MediaType.TEXT_PLAIN_UTF_8)
+    public Response getStatusForLoadBalancer() {
+        Set<LoadBalancerCheckProvider> healthStatusProviders = session.getAllProviders(LoadBalancerCheckProvider.class);
+        if (healthStatusProviders.stream().anyMatch(LoadBalancerCheckProvider::isDown)) {
+            return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity("DOWN").build();
+        } else {
+            return Response.ok().entity("UP").build();
+        }
+    }
+
+}