Supported option to specify site name for multi-site deployments

Closes #26460

Signed-off-by: Václav Muzikář <vmuzikar@redhat.com>
Signed-off-by: Alexander Schwartz <aschwart@redhat.com>
Co-authored-by: Alexander Schwartz <aschwart@redhat.com>

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -46,9 +46,12 @@ import org.keycloak.models.cache.infinispan.events.RealmUpdatedEvent;
 import org.keycloak.models.utils.KeycloakModelUtils;
 import org.keycloak.models.utils.PostMigrationEvent;
 import org.keycloak.provider.InvalidationHandler.ObjectType;
+import org.keycloak.provider.ProviderConfigProperty;
+import org.keycloak.provider.ProviderConfigurationBuilder;
 import org.keycloak.provider.ProviderEvent;
 
 import java.util.Iterator;
+import java.util.List;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
@@ -225,7 +228,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, getRevisionCacheConfig(authzRevisionsMaxEntries));
         cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, true);
 
-        this.topologyInfo = new TopologyInfo(cacheManager, config, false);
+        this.topologyInfo = new TopologyInfo(cacheManager, config, false, getId());
 
         logger.debugv("Using container managed Infinispan cache container, lookup={0}", cacheManager);
 
@@ -239,7 +242,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         boolean async = config.getBoolean("async", false);
         boolean useKeycloakTimeService = config.getBoolean("useKeycloakTimeService", false);
 
-        this.topologyInfo = new TopologyInfo(cacheManager, config, true);
+        this.topologyInfo = new TopologyInfo(cacheManager, config, true, getId());
 
         if (clustered) {
             String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));

model/infinispan/src/main/java/org/keycloak/connections/infinispan/TopologyInfo.java

@@ -20,7 +20,6 @@ package org.keycloak.connections.infinispan;
 import java.net.InetSocketAddress;
 import java.security.SecureRandom;
 import java.util.Objects;
-import java.util.Optional;
 
 import org.infinispan.Cache;
 import org.infinispan.distribution.DistributionManager;
@@ -55,22 +54,28 @@ public class TopologyInfo {
     private final boolean isGeneratedNodeName;
 
 
-    public TopologyInfo(EmbeddedCacheManager cacheManager, Config.Scope config, boolean embedded) {
+    public TopologyInfo(EmbeddedCacheManager cacheManager, Config.Scope config, boolean embedded, String providerId) {
         String siteName;
         String nodeName;
         boolean isGeneratedNodeName = false;
 
+        if (System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME) != null) {
+            throw new IllegalArgumentException(
+                    String.format("System property %s is in use. Use --spi-connections-infinispan-%s-site-name config option instead",
+                            InfinispanConnectionProvider.JBOSS_SITE_NAME, providerId));
+        }
+
         if (!embedded) {
             Transport transport = cacheManager.getTransport();
             if (transport != null) {
                 nodeName = transport.getAddress().toString();
                 siteName = cacheManager.getCacheManagerConfiguration().transport().siteId();
                 if (siteName == null) {
-                    siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
+                    siteName = config.get("siteName");
                 }
             } else {
                 nodeName = System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME);
-                siteName = System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME);
+                siteName = config.get("siteName");
             }
             if (nodeName == null || nodeName.equals("localhost")) {
                 isGeneratedNodeName = true;
@@ -84,7 +89,7 @@ public class TopologyInfo {
                 nodeName = null;
             }
 
-            siteName = config.get("siteName", System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME));
+            siteName = config.get("siteName");
             if (siteName != null && siteName.isEmpty()) {
                 siteName = null;
             }

quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/legacy/infinispan/QuarkusInfinispanConnectionFactory.java

@@ -19,6 +19,10 @@ package org.keycloak.quarkus.runtime.storage.legacy.infinispan;
 
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.keycloak.connections.infinispan.DefaultInfinispanConnectionProviderFactory;
+import org.keycloak.provider.ProviderConfigProperty;
+import org.keycloak.provider.ProviderConfigurationBuilder;
+
+import java.util.List;
 
 /**
  * @author <a href="mailto:psilva@redhat.com">Pedro Igor</a>
@@ -43,4 +47,15 @@ public class QuarkusInfinispanConnectionFactory extends DefaultInfinispanConnect
     public String getId() {
         return "quarkus";
     }
+
+    @Override
+    public List<ProviderConfigProperty> getConfigMetadata() {
+        return ProviderConfigurationBuilder.create()
+                .property()
+                .name("site-name")
+                .helpText("Site name for multi-site deployments")
+                .type("string")
+                .add()
+                .build();
+    }
 }

quarkus/tests/integration/src/test/java/org/keycloak/it/storage/database/ExternalInfinispanTest.java

@@ -18,8 +18,10 @@
 package org.keycloak.it.storage.database;
 
 import io.quarkus.test.junit.main.Launch;
+import io.quarkus.test.junit.main.LaunchResult;
 import org.junit.jupiter.api.Test;
 import org.keycloak.common.util.Retry;
+import org.keycloak.it.junit5.extension.CLIResult;
 import org.keycloak.it.junit5.extension.DistributionTest;
 import org.keycloak.it.junit5.extension.InfinispanContainer;
 import org.keycloak.it.junit5.extension.WithExternalInfinispan;
@@ -31,12 +33,12 @@ import static io.restassured.RestAssured.when;
 public class ExternalInfinispanTest {
 
     @Test
-    @Launch({ "start-dev", "--features=multi-site", "--cache=ispn", "--cache-config-file=../../../test-classes/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml", "-Djboss.site.name=ISPN" })
+    @Launch({ "start-dev", "--features=multi-site", "--cache=ispn", "--cache-config-file=../../../test-classes/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml", "--spi-connections-infinispan-quarkus-site-name=ISPN" })
     void testLoadBalancerCheckFailure() {
         when().get("/lb-check").then()
                 .statusCode(200);
 
-        InfinispanContainer.remoteCacheManager.administration().removeCache("sessions");
+        InfinispanContainer.removeCache("sessions");
 
         // The `lb-check` relies on the Infinispan's persistence check status. By default, Infinispan checks in the background every second that the remote store is available.
         // So we'll wait on average about one second here for the check to switch its state.
@@ -45,4 +47,10 @@ public class ExternalInfinispanTest {
                     .statusCode(503);
         }, 10, 200);
     }
+
+    @Test
+    @Launch({ "start-dev", "--features=multi-site", "--cache=ispn", "--cache-config-file=../../../test-classes/ExternalInfinispan/kcb-infinispan-cache-remote-store-config.xml", "-Djboss.site.name=ISPN" })
+    void testSiteNameAsSystemProperty(LaunchResult result) {
+        ((CLIResult) result).assertMessage("System property jboss.site.name is in use. Use --spi-connections-infinispan-quarkus-site-name config option instead");
+    }
 }

quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/CLITestExtension.java

@@ -199,7 +199,7 @@ public class CLITestExtension extends QuarkusMainTestExtension {
             databaseContainer.stop();
             databaseContainer = null;
         }
-        if (infinispanContainer != null && infinispanContainer.isRunning()) {
+        if (infinispanContainer != null) {
             infinispanContainer.stop();
         }
         result = null;

quarkus/tests/junit5/src/main/java/org/keycloak/it/junit5/extension/InfinispanContainer.java

@@ -67,6 +67,13 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
         return INFINISPAN_IMAGE;
     }
 
+    public static void removeCache(String cache) {
+        // first stop the cache to avoid leaking MBeans for the HotRodClient
+        // see: https://issues.redhat.com/browse/ISPN-15606
+        remoteCacheManager.getCache(cache).stop();
+        remoteCacheManager.administration().removeCache(cache);
+    }
+
     private void establishHotRodConnection() {
         ConfigurationBuilder configBuilder = new ConfigurationBuilder()
                 .addServers(getContainerIpAddress() + ":11222")
@@ -84,6 +91,8 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
 
     @Override
     public void start() {
+        logger().info("Starting ISPN container");
+
         super.start();
 
         establishHotRodConnection();
@@ -95,6 +104,17 @@ public class InfinispanContainer extends GenericContainer<InfinispanContainer> {
                 });
     }
 
+    @Override
+    public void stop() {
+        logger().info("Stopping ISPN container");
+
+        if (remoteCacheManager != null) {
+            remoteCacheManager.stop();
+        }
+
+        super.stop();
+    }
+
     public void createCache(RemoteCacheManager remoteCacheManager, String cacheName) {
         String xml = String.format("<distributed-cache name=\"%s\" mode=\"SYNC\" owners=\"2\"></distributed-cache>" , cacheName);
         remoteCacheManager.administration().getOrCreateCache(cacheName, new XMLStringConfiguration(xml));

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -185,7 +185,7 @@
         "default": {
             "jgroupsUdpMcastAddr": "${keycloak.connectionsInfinispan.jgroupsUdpMcastAddr:234.56.78.90}",
             "nodeName": "${keycloak.connectionsInfinispan.nodeName,jboss.node.name:}",
-            "siteName": "${keycloak.connectionsInfinispan.siteName,jboss.site.name:}",
+            "siteName": "${keycloak.connectionsInfinispan.siteName:}",
             "clustered": "${keycloak.connectionsInfinispan.clustered:false}",
             "async": "${keycloak.connectionsInfinispan.async:false}",
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -519,7 +519,7 @@
                     -Djboss.socket.binding.port-offset=${auth.server.crossdc01.port.offset}
                     -Djboss.default.multicast.address=234.56.78.1
                     -Dremote.cache.port=12232
-                    -Djboss.site.name=dc0
+                    -Dkeycloak.connectionsInfinispan.siteName=dc0
                     -Djboss.node.name=auth-server-${node.name}-cross-dc-0_1
                     -Dauth.server.truststore=${auth.server.truststore}
                     -Dauth.server.truststore.password=${auth.server.truststore.password}
@@ -551,7 +551,7 @@
                     -Djboss.socket.binding.port-offset=${auth.server.crossdc02.port.offset}
                     -Djboss.default.multicast.address=234.56.78.1
                     -Dremote.cache.port=12232
-                    -Djboss.site.name=dc0
+                    -Dkeycloak.connectionsInfinispan.siteName=dc0
                     -Djboss.node.name=auth-server-${node.name}-cross-dc-0_2-manual
                     -Dauth.server.truststore=${auth.server.truststore}
                     -Dauth.server.truststore.password=${auth.server.truststore.password}
@@ -584,7 +584,7 @@
                     -Djboss.socket.binding.port-offset=${auth.server.crossdc11.port.offset}
                     -Djboss.default.multicast.address=234.56.78.2
                     -Dremote.cache.port=13232
-                    -Djboss.site.name=dc1
+                    -Dkeycloak.connectionsInfinispan.siteName=dc1
                     -Djboss.node.name=auth-server-${node.name}-cross-dc-1_1
                     -Dauth.server.truststore=${auth.server.truststore}
                     -Dauth.server.truststore.password=${auth.server.truststore.password}
@@ -616,7 +616,7 @@
                     -Djboss.socket.binding.port-offset=${auth.server.crossdc12.port.offset}
                     -Djboss.default.multicast.address=234.56.78.2
                     -Dremote.cache.port=13232
-                    -Djboss.site.name=dc1
+                    -Dkeycloak.connectionsInfinispan.siteName=dc1
                     -Djboss.node.name=auth-server-${node.name}-cross-dc-1_2-manual
                     -Dauth.server.truststore=${auth.server.truststore}
                     -Dauth.server.truststore.password=${auth.server.truststore.password}

testsuite/utils/src/main/resources/META-INF/keycloak-server.json

@@ -137,7 +137,7 @@
         "default": {
             "jgroupsUdpMcastAddr": "${keycloak.connectionsInfinispan.jgroupsUdpMcastAddr:234.56.78.90}",
             "nodeName": "${keycloak.connectionsInfinispan.nodeName,jboss.node.name:}",
-            "siteName": "${keycloak.connectionsInfinispan.siteName,jboss.site.name:}",
+            "siteName": "${keycloak.connectionsInfinispan.siteName:}",
             "clustered": "${keycloak.connectionsInfinispan.clustered:}",
             "async": "${keycloak.connectionsInfinispan.async:}",
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:}",