From 53f156ec83a45c1dd4e04ead1eda593b71b8b475 Mon Sep 17 00:00:00 2001 From: Pedro Igor Date: Tue, 19 Nov 2019 13:05:15 -0300 Subject: [PATCH] [KEYCLOAK-11328] - Initial Server.x Clustering Configuration --- distribution/server-x/assembly.xml | 7 ++ distribution/server-x/pom.xml | 2 +- .../server-x/src/main/content/bin/kc.sh | 2 +- .../src/main/content/conf/cluster.xml | 105 ++++++++++++++++++ ...ltInfinispanConnectionProviderFactory.java | 83 ++++++++------ quarkus/extensions/pom.xml | 8 ++ .../quarkus/QuarkusCacheManagerProvider.java | 101 +++++++++++++++++ ...ycloak.cluster.ManagedCacheManagerProvider | 1 + quarkus/server/pom.xml | 1 - .../resources/META-INF/keycloak-server.json | 3 +- .../src/main/resources/META-INF/web.xml | 6 - .../resources/default-clustered-cache.xml | 61 ++++++++++ .../main/resources/default-local-cache.xml | 63 +++++++++++ .../cluster/ManagedCacheManagerProvider.java | 30 +++++ .../wildfly/WildflyCacheManagerProvider.java | 44 ++++++++ ...ycloak.cluster.ManagedCacheManagerProvider | 18 +++ 16 files changed, 489 insertions(+), 46 deletions(-) create mode 100644 distribution/server-x/src/main/content/conf/cluster.xml create mode 100644 quarkus/extensions/src/main/java/org/keycloak/provider/quarkus/QuarkusCacheManagerProvider.java create mode 100644 quarkus/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider create mode 100644 quarkus/server/src/main/resources/default-clustered-cache.xml create mode 100644 quarkus/server/src/main/resources/default-local-cache.xml create mode 100644 server-spi-private/src/main/java/org/keycloak/cluster/ManagedCacheManagerProvider.java create mode 100644 wildfly/extensions/src/main/java/org/keycloak/provider/wildfly/WildflyCacheManagerProvider.java create mode 100644 wildfly/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider diff --git a/distribution/server-x/assembly.xml b/distribution/server-x/assembly.xml index fe6952edbe..2ce052bd13 100755 --- a/distribution/server-x/assembly.xml +++ b/distribution/server-x/assembly.xml @@ -75,6 +75,13 @@ keycloak-runner.jar + + src/main/content/conf + conf + + *.* + + ../../ diff --git a/distribution/server-x/pom.xml b/distribution/server-x/pom.xml index 6d5ec97711..b982df8588 100755 --- a/distribution/server-x/pom.xml +++ b/distribution/server-x/pom.xml @@ -21,7 +21,7 @@ keycloak-distribution-parent org.keycloak - 8.0.0-SNAPSHOT + 9.0.0-SNAPSHOT keycloak-server-x diff --git a/distribution/server-x/src/main/content/bin/kc.sh b/distribution/server-x/src/main/content/bin/kc.sh index 52560a3f57..919efa24b4 100644 --- a/distribution/server-x/src/main/content/bin/kc.sh +++ b/distribution/server-x/src/main/content/bin/kc.sh @@ -21,4 +21,4 @@ if [ "x$RESOLVED_NAME" = "x" ]; then fi DIRNAME=`dirname "$RESOLVED_NAME"` -java -Dkeycloak.theme.dir=$DIRNAME/../themes -cp "$DIRNAME/../providers/*:$DIRNAME/../lib/keycloak-runner.jar" io.quarkus.runner.GeneratedMain "$@" +java -Dkeycloak.home.dir=$DIRNAME/../ -Dkeycloak.theme.dir=$DIRNAME/../themes -Djava.net.preferIPv4Stack=true -cp "$DIRNAME/../providers/*:$DIRNAME/../lib/keycloak-runner.jar" io.quarkus.runner.GeneratedMain "$@" diff --git a/distribution/server-x/src/main/content/conf/cluster.xml b/distribution/server-x/src/main/content/conf/cluster.xml new file mode 100644 index 0000000000..870142877f --- /dev/null +++ b/distribution/server-x/src/main/content/conf/cluster.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java index 467b03fa3e..00f54d2968 100755 --- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java @@ -17,6 +17,8 @@ package org.keycloak.connections.infinispan; +import java.util.Iterator; +import java.util.ServiceLoader; import java.util.concurrent.TimeUnit; import org.infinispan.client.hotrod.ProtocolVersion; @@ -37,11 +39,11 @@ import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup; import org.jboss.logging.Logger; import org.jgroups.JChannel; import org.keycloak.Config; +import org.keycloak.cluster.ManagedCacheManagerProvider; import org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; -import javax.naming.InitialContext; import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder; /** @@ -98,11 +100,24 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon if (cacheManager == null) { synchronized (this) { if (cacheManager == null) { - String cacheContainer = config.get("cacheContainer"); - if (cacheContainer != null) { - initContainerManaged(cacheContainer); - } else { + EmbeddedCacheManager managedCacheManager = null; + Iterator providers = ServiceLoader.load(ManagedCacheManagerProvider.class) + .iterator(); + + if (providers.hasNext()) { + ManagedCacheManagerProvider provider = providers.next(); + + if (providers.hasNext()) { + throw new RuntimeException("Multiple " + org.keycloak.cluster.ManagedCacheManagerProvider.class + " providers found."); + } + + managedCacheManager = provider.getCacheManager(config); + } + + if (managedCacheManager == null) { initEmbedded(); + } else { + initContainerManaged(managedCacheManager); } logger.infof(topologyInfo.toString()); @@ -113,45 +128,41 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon } } - protected void initContainerManaged(String cacheContainerLookup) { - try { - cacheManager = (EmbeddedCacheManager) new InitialContext().lookup(cacheContainerLookup); - containerManaged = true; + protected void initContainerManaged(EmbeddedCacheManager cacheManager) { + this.cacheManager = cacheManager; + containerManaged = true; - long realmRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().memory().size(); - realmRevisionsMaxEntries = realmRevisionsMaxEntries > 0 - ? 2 * realmRevisionsMaxEntries - : InfinispanConnectionProvider.REALM_REVISIONS_CACHE_DEFAULT_MAX; + long realmRevisionsMaxEntries = this.cacheManager.getCache(InfinispanConnectionProvider.REALM_CACHE_NAME).getCacheConfiguration().memory().size(); + realmRevisionsMaxEntries = realmRevisionsMaxEntries > 0 + ? 2 * realmRevisionsMaxEntries + : InfinispanConnectionProvider.REALM_REVISIONS_CACHE_DEFAULT_MAX; - cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, getRevisionCacheConfig(realmRevisionsMaxEntries)); - cacheManager.getCache(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, true); + this.cacheManager.defineConfiguration(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, getRevisionCacheConfig(realmRevisionsMaxEntries)); + this.cacheManager.getCache(InfinispanConnectionProvider.REALM_REVISIONS_CACHE_NAME, true); - long userRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().memory().size(); - userRevisionsMaxEntries = userRevisionsMaxEntries > 0 - ? 2 * userRevisionsMaxEntries - : InfinispanConnectionProvider.USER_REVISIONS_CACHE_DEFAULT_MAX; + long userRevisionsMaxEntries = this.cacheManager.getCache(InfinispanConnectionProvider.USER_CACHE_NAME).getCacheConfiguration().memory().size(); + userRevisionsMaxEntries = userRevisionsMaxEntries > 0 + ? 2 * userRevisionsMaxEntries + : InfinispanConnectionProvider.USER_REVISIONS_CACHE_DEFAULT_MAX; - cacheManager.defineConfiguration(InfinispanConnectionProvider.USER_REVISIONS_CACHE_NAME, getRevisionCacheConfig(userRevisionsMaxEntries)); - cacheManager.getCache(InfinispanConnectionProvider.USER_REVISIONS_CACHE_NAME, true); - cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME, true); - cacheManager.getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME, true); - cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true); - cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true); + this.cacheManager.defineConfiguration(InfinispanConnectionProvider.USER_REVISIONS_CACHE_NAME, getRevisionCacheConfig(userRevisionsMaxEntries)); + this.cacheManager.getCache(InfinispanConnectionProvider.USER_REVISIONS_CACHE_NAME, true); + this.cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME, true); + this.cacheManager.getCache(InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME, true); + this.cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true); + this.cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true); - long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().memory().size(); - authzRevisionsMaxEntries = authzRevisionsMaxEntries > 0 - ? 2 * authzRevisionsMaxEntries - : InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX; + long authzRevisionsMaxEntries = this.cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().memory().size(); + authzRevisionsMaxEntries = authzRevisionsMaxEntries > 0 + ? 2 * authzRevisionsMaxEntries + : InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX; - cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, getRevisionCacheConfig(authzRevisionsMaxEntries)); - cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, true); + this.cacheManager.defineConfiguration(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, getRevisionCacheConfig(authzRevisionsMaxEntries)); + this.cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME, true); - this.topologyInfo = new TopologyInfo(cacheManager, config, false); + this.topologyInfo = new TopologyInfo(this.cacheManager, config, false); - logger.debugv("Using container managed Infinispan cache container, lookup={0}", cacheContainerLookup); - } catch (Exception e) { - throw new RuntimeException("Failed to retrieve cache container", e); - } + logger.debugv("Using container managed Infinispan cache container, lookup={0}", cacheManager); } protected void initEmbedded() { diff --git a/quarkus/extensions/pom.xml b/quarkus/extensions/pom.xml index 4bab3990d6..16516b173b 100644 --- a/quarkus/extensions/pom.xml +++ b/quarkus/extensions/pom.xml @@ -36,6 +36,14 @@ org.keycloak keycloak-services + + org.keycloak + keycloak-server-spi-private + + + org.infinispan + infinispan-core + io.quarkus quarkus-arc diff --git a/quarkus/extensions/src/main/java/org/keycloak/provider/quarkus/QuarkusCacheManagerProvider.java b/quarkus/extensions/src/main/java/org/keycloak/provider/quarkus/QuarkusCacheManagerProvider.java new file mode 100644 index 0000000000..5c0acbd357 --- /dev/null +++ b/quarkus/extensions/src/main/java/org/keycloak/provider/quarkus/QuarkusCacheManagerProvider.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.provider.quarkus; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.infinispan.commons.util.FileLookupFactory; +import org.infinispan.configuration.parsing.ConfigurationBuilderHolder; +import org.infinispan.configuration.parsing.ParserRegistry; +import org.infinispan.manager.DefaultCacheManager; +import org.jboss.logging.Logger; +import org.keycloak.Config; +import org.keycloak.cluster.ManagedCacheManagerProvider; + +/** + * @author Pedro Igor + */ +public final class QuarkusCacheManagerProvider implements ManagedCacheManagerProvider { + + private static final Logger log = Logger.getLogger(QuarkusCacheManagerProvider.class); + + private static final String DEFAULT_CONFIGURATION_FILE_NAME = "cluster.xml"; + + @Override + public C getCacheManager(Config.Scope config) { + try { + InputStream configurationStream = loadConfiguration(config); + ConfigurationBuilderHolder builder = new ParserRegistry().parse(configurationStream); + configureTransportStack(config, builder); + return (C) new DefaultCacheManager(builder, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private InputStream loadConfiguration(Config.Scope config) throws FileNotFoundException { + String homeDir = System.getProperty("keycloak.home.dir"); + + if (homeDir == null) { + log.warn("Keycloak home directory not set."); + return loadDefaultConfiguration(config); + } + + Path configPath = Paths.get(homeDir + "/conf/" + getConfigFileName(config)); + + if (configPath.toFile().exists()) { + log.debugf("Loading cluster configuration from %s", configPath); + return FileLookupFactory.newInstance() + .lookupFileStrict(configPath.toUri(), Thread.currentThread().getContextClassLoader()); + } + + log.infof("Clustering configuration file not found at %s.", configPath); + + return loadDefaultConfiguration(config); + } + + private InputStream loadDefaultConfiguration(Config.Scope config) throws FileNotFoundException { + if (config.getBoolean("clustered")) { + log.debugf("Using default clustered cache configuration."); + return FileLookupFactory.newInstance() + .lookupFileStrict("default-clustered-cache.xml", Thread.currentThread().getContextClassLoader()); + } + + log.debug("Using default local cache configuration."); + + return FileLookupFactory.newInstance() + .lookupFileStrict("default-local-cache.xml", Thread.currentThread().getContextClassLoader()); + } + + private String getConfigFileName(Config.Scope config) { + String configFile = config.get("configFile"); + return configFile == null ? DEFAULT_CONFIGURATION_FILE_NAME : configFile; + } + + private void configureTransportStack(Config.Scope config, ConfigurationBuilderHolder builder) { + String transportStack = config.get("stack"); + + if (transportStack != null) { + builder.getGlobalConfigurationBuilder().transport().defaultTransport() + .addProperty("configurationFile", "default-configs/default-jgroups-" + transportStack + ".xml"); + } + } +} diff --git a/quarkus/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider b/quarkus/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider new file mode 100644 index 0000000000..9365509a2b --- /dev/null +++ b/quarkus/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider @@ -0,0 +1 @@ +org.keycloak.provider.quarkus.QuarkusCacheManagerProvider \ No newline at end of file diff --git a/quarkus/server/pom.xml b/quarkus/server/pom.xml index adea690df8..9e76e2bce7 100644 --- a/quarkus/server/pom.xml +++ b/quarkus/server/pom.xml @@ -316,7 +316,6 @@ - org.infinispan infinispan-core diff --git a/quarkus/server/src/main/resources/META-INF/keycloak-server.json b/quarkus/server/src/main/resources/META-INF/keycloak-server.json index 5ed3ac19af..9abad69c8b 100755 --- a/quarkus/server/src/main/resources/META-INF/keycloak-server.json +++ b/quarkus/server/src/main/resources/META-INF/keycloak-server.json @@ -105,7 +105,8 @@ "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}", "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}", "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}", - "hotrodProtocolVersion": "${keycloak.connectionsInfinispan.hotrodProtocolVersion}" + "hotrodProtocolVersion": "${keycloak.connectionsInfinispan.hotrodProtocolVersion}", + "stack": "${keycloak.connectionsInfinispan.stack:udp}" } }, diff --git a/quarkus/server/src/main/resources/META-INF/web.xml b/quarkus/server/src/main/resources/META-INF/web.xml index 26a88ff94f..09fe42b57b 100644 --- a/quarkus/server/src/main/resources/META-INF/web.xml +++ b/quarkus/server/src/main/resources/META-INF/web.xml @@ -67,10 +67,4 @@ Keycloak Session Management /* - - - infinispan/Keycloak - org.infinispan.manager.EmbeddedCacheManager - java:jboss/infinispan/container/keycloak - diff --git a/quarkus/server/src/main/resources/default-clustered-cache.xml b/quarkus/server/src/main/resources/default-clustered-cache.xml new file mode 100644 index 0000000000..17f9484084 --- /dev/null +++ b/quarkus/server/src/main/resources/default-clustered-cache.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/quarkus/server/src/main/resources/default-local-cache.xml b/quarkus/server/src/main/resources/default-local-cache.xml new file mode 100644 index 0000000000..fd2a4b45f4 --- /dev/null +++ b/quarkus/server/src/main/resources/default-local-cache.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server-spi-private/src/main/java/org/keycloak/cluster/ManagedCacheManagerProvider.java b/server-spi-private/src/main/java/org/keycloak/cluster/ManagedCacheManagerProvider.java new file mode 100644 index 0000000000..2f32bf3476 --- /dev/null +++ b/server-spi-private/src/main/java/org/keycloak/cluster/ManagedCacheManagerProvider.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.cluster; + +import org.keycloak.Config; + +/** + * A Service Provider Interface (SPI) that allows to plug-in a cache manager instance. + * + * @author Pedro Igor + */ +public interface ManagedCacheManagerProvider { + + C getCacheManager(Config.Scope config); +} diff --git a/wildfly/extensions/src/main/java/org/keycloak/provider/wildfly/WildflyCacheManagerProvider.java b/wildfly/extensions/src/main/java/org/keycloak/provider/wildfly/WildflyCacheManagerProvider.java new file mode 100644 index 0000000000..11de009cc2 --- /dev/null +++ b/wildfly/extensions/src/main/java/org/keycloak/provider/wildfly/WildflyCacheManagerProvider.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.provider.wildfly; + +import javax.naming.InitialContext; + +import org.keycloak.Config; +import org.keycloak.cluster.ManagedCacheManagerProvider; + +/** + * @author Pedro Igor + */ +public class WildflyCacheManagerProvider implements ManagedCacheManagerProvider { + + @Override + public C getCacheManager(Config.Scope config) { + String cacheContainer = config.get("cacheContainer"); + + if (cacheContainer == null) { + return null; + } + + try { + return (C) new InitialContext().lookup(cacheContainer); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve cache container", e); + } + } +} diff --git a/wildfly/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider b/wildfly/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider new file mode 100644 index 0000000000..a1836f87ab --- /dev/null +++ b/wildfly/extensions/src/main/resources/META-INF/services/org.keycloak.cluster.ManagedCacheManagerProvider @@ -0,0 +1,18 @@ +# +# Copyright 2019 Red Hat, Inc. and/or its affiliates +# and other contributors as indicated by the @author tags. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.keycloak.provider.wildfly.WildflyCacheManagerProvider \ No newline at end of file