From ca92bcbf7ff732a6f0e3e17ffd62a588fbaccfd1 Mon Sep 17 00:00:00 2001 From: mposolda Date: Tue, 12 Sep 2017 12:18:24 +0200 Subject: [PATCH] KEYCLOAK-5480 Cross-DC setup: Remote cache stores are connecting to Infinispan servers in both datacenters --- misc/CrossDataCenter.md | 57 +++++---- ...ltInfinispanConnectionProviderFactory.java | 38 +++--- .../remotestore/KeycloakRemoteStore.java | 35 ++++- .../KeycloakRemoteStoreConfiguration.java | 17 ++- ...ycloakRemoteStoreConfigurationBuilder.java | 67 +++++++++- .../KeycloakTcpTransportFactory.java | 120 ++++++++++++++++++ .../base/src/test/resources/log4j.properties | 2 + .../resources/META-INF/keycloak-server.json | 2 +- 8 files changed, 288 insertions(+), 50 deletions(-) create mode 100644 model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakTcpTransportFactory.java diff --git a/misc/CrossDataCenter.md b/misc/CrossDataCenter.md index 261d099500..7d2d8418d2 100644 --- a/misc/CrossDataCenter.md +++ b/misc/CrossDataCenter.md @@ -110,32 +110,24 @@ Keycloak servers setup ``` -3.2) Add output-socket-binding for `remote-cache` under `socket-binding-group` element: - -```xml - - ... - - - - - -``` - -3.3) Add this `module` attribute under `cache-container` element of name `keycloak` : +3.2) Add this `module` attribute under `cache-container` element of name `keycloak` : ```xml ``` -3.4) Add the `remote-store` under `work` cache: +3.3) Add the `store` under `work` cache: ```xml - - true + + true org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory - + org.keycloak.models.sessions.infinispan.remotestore.KeycloakTcpTransportFactory + localhost:${remote.cache.port} + work + false + ``` @@ -146,17 +138,19 @@ Keycloak servers setup sessions work + true ``` -3.6) Same for `offlineSessions` and `loginFailures` caches: +3.6) Same for `offlineSessions` and `loginFailures` caches (The only difference from `sessions` cache is, that `remoteCacheName` property value are different: ```xml offlineSessions work + true @@ -164,13 +158,28 @@ Keycloak servers setup loginFailures work + true ``` -3.7) The configuration of distributed cache `authenticationSessions` and other caches is left unchanged. +3.7) The configuration of `actionTokens` cache have different `remoteCacheName`, `sessionCache` and the `preload` attribute: -3.8) Optionally enable DEBUG logging under `logging` subsystem: +```xml + + + + + actionTokens + work + false + + +``` + +3.8) The configuration of distributed cache `authenticationSessions` and other caches is left unchanged. + +3.9) Optionally enable DEBUG logging under `logging` subsystem: ```xml @@ -211,7 +220,7 @@ cd NODE12/bin The cluster nodes should be connected. This should be in the log of both NODE11 and NODE12: ``` -Received new cluster view for channel hibernate: [node11|1] (2) [node11, node12] +Received new cluster view for channel keycloak: [node11|1] (2) [node11, node12] ``` 7) Start `NODE21` : @@ -226,7 +235,7 @@ cd NODE21/bin It shouldn't be connected to the cluster with `NODE11` and `NODE12`, but to separate one: ``` -Received new cluster view for channel hibernate: [node21|0] (1) [node21] +Received new cluster view for channel keycloak: [node21|0] (1) [node21] ``` 8) Start `NODE22` : @@ -241,7 +250,7 @@ cd NODE22/bin It should be in cluster with `NODE21` : ``` -Received new cluster view for channel server: [node21|1] (2) [node21, node22] +Received new cluster view for channel keycloak: [node21|1] (2) [node21, node22] ``` 9) Test: @@ -263,5 +272,5 @@ the same sessions in tab `Sessions` of particular user, client or realm on all 4 Event 'CLIENT_CACHE_ENTRY_REMOVED', key '193489e7-e2bc-4069-afe8-f1dfa73084ea', skip 'false' ``` -This is just a starting point and the instructions are subject to change. We plan performance improvements especially around performance. If you +This is just a starting point and the instructions are subject to change. We plan various improvements especially around performance. If you have any feedback regarding cross-dc scenario, please let us know on keycloak-user mailing list referred from [Keycloak home page](http://www.keycloak.org/community.html). \ No newline at end of file diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java index 5135e90bea..24ef8737bb 100755 --- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java @@ -30,7 +30,6 @@ import org.infinispan.eviction.EvictionStrategy; import org.infinispan.eviction.EvictionType; import org.infinispan.manager.DefaultCacheManager; import org.infinispan.manager.EmbeddedCacheManager; -import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder; import org.infinispan.remoting.transport.Transport; import org.infinispan.remoting.transport.jgroups.JGroupsTransport; import org.infinispan.transaction.LockingMode; @@ -43,6 +42,7 @@ import org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory; import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSessionFactory; import org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder; +import org.keycloak.models.sessions.infinispan.remotestore.KeycloakTcpTransportFactory; import javax.naming.InitialContext; @@ -246,7 +246,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon if (jdgEnabled) { sessionConfigBuilder = new ConfigurationBuilder(); sessionConfigBuilder.read(sessionCacheConfigurationBase); - configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.SESSION_CACHE_NAME, KeycloakRemoteStoreConfigurationBuilder.class); + configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.SESSION_CACHE_NAME, true); } Configuration sessionCacheConfiguration = sessionConfigBuilder.build(); cacheManager.defineConfiguration(InfinispanConnectionProvider.SESSION_CACHE_NAME, sessionCacheConfiguration); @@ -254,7 +254,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon if (jdgEnabled) { sessionConfigBuilder = new ConfigurationBuilder(); sessionConfigBuilder.read(sessionCacheConfigurationBase); - configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME, KeycloakRemoteStoreConfigurationBuilder.class); + configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME, true); } sessionCacheConfiguration = sessionConfigBuilder.build(); cacheManager.defineConfiguration(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME, sessionCacheConfiguration); @@ -262,7 +262,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon if (jdgEnabled) { sessionConfigBuilder = new ConfigurationBuilder(); sessionConfigBuilder.read(sessionCacheConfigurationBase); - configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME, KeycloakRemoteStoreConfigurationBuilder.class); + configureRemoteCacheStore(sessionConfigBuilder, async, InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME, true); } sessionCacheConfiguration = sessionConfigBuilder.build(); cacheManager.defineConfiguration(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME, sessionCacheConfiguration); @@ -281,7 +281,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon } if (jdgEnabled) { - configureRemoteCacheStore(replicationConfigBuilder, async, InfinispanConnectionProvider.WORK_CACHE_NAME, RemoteStoreConfigurationBuilder.class); + configureRemoteCacheStore(replicationConfigBuilder, async, InfinispanConnectionProvider.WORK_CACHE_NAME, false); } Configuration replicationEvictionCacheConfiguration = replicationConfigBuilder.build(); @@ -349,13 +349,15 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon } // Used for cross-data centers scenario. Usually integration with external JDG server, which itself handles communication between DCs. - private void configureRemoteCacheStore(ConfigurationBuilder builder, boolean async, String cacheName, Class configBuilderClass) { - String jdgServer = config.get("remoteStoreServer", "localhost"); + private void configureRemoteCacheStore(ConfigurationBuilder builder, boolean async, String cacheName, boolean sessionCache) { + String jdgServer = config.get("remoteStoreHost", "localhost"); Integer jdgPort = config.getInt("remoteStorePort", 11222); builder.persistence() .passivation(false) - .addStore(configBuilderClass) + .addStore(KeycloakRemoteStoreConfigurationBuilder.class) + .remoteServers(jdgServer + ":" + jdgPort) + .sessionCache(sessionCache) .fetchPersistentState(false) .ignoreModifications(false) .purgeOnStartup(false) @@ -365,9 +367,10 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon .rawValues(true) .forceReturnValues(false) .marshaller(KeycloakHotRodMarshallerFactory.class.getName()) - .addServer() - .host(jdgServer) - .port(jdgPort) + .transportFactory(KeycloakTcpTransportFactory.class.getName()) +// .addServer() +// .host(jdgServer) +// .port(jdgPort) // .connectionPool() // .maxActive(100) // .exhaustedAction(ExhaustedAction.CREATE_NEW) @@ -377,12 +380,14 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon } private void configureRemoteActionTokenCacheStore(ConfigurationBuilder builder, boolean async) { - String jdgServer = config.get("remoteStoreServer", "localhost"); + String jdgServer = config.get("remoteStoreHost", "localhost"); Integer jdgPort = config.getInt("remoteStorePort", 11222); builder.persistence() .passivation(false) - .addStore(RemoteStoreConfigurationBuilder.class) + .addStore(KeycloakRemoteStoreConfigurationBuilder.class) + .remoteServers(jdgServer + ":" + jdgPort) + .sessionCache(false) .fetchPersistentState(false) .ignoreModifications(false) .purgeOnStartup(false) @@ -392,9 +397,10 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon .rawValues(true) .forceReturnValues(false) .marshaller(KeycloakHotRodMarshallerFactory.class.getName()) - .addServer() - .host(jdgServer) - .port(jdgPort) + .transportFactory(KeycloakTcpTransportFactory.class.getName()) +// .addServer() +// .host(jdgServer) +// .port(jdgPort) .async() .enabled(async); diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStore.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStore.java index a0f5e42379..88df0492e6 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStore.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStore.java @@ -32,7 +32,6 @@ import org.infinispan.metadata.InternalMetadata; import org.infinispan.persistence.InitializationContextImpl; import org.infinispan.persistence.remote.RemoteStore; import org.infinispan.persistence.remote.configuration.RemoteStoreConfiguration; -import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder; import org.infinispan.persistence.spi.InitializationContext; import org.infinispan.persistence.spi.PersistenceException; import org.jboss.logging.Logger; @@ -52,6 +51,7 @@ public class KeycloakRemoteStore extends RemoteStore { @Override public void start() throws PersistenceException { this.remoteCacheName = getConfiguration().remoteCacheName(); + Boolean sessionCache = getConfiguration().sessionCache(); String cacheTemplateName = getConfiguration().useConfigTemplateFromCache(); @@ -64,7 +64,7 @@ public class KeycloakRemoteStore extends RemoteStore { Optional optional = cacheManager.getCacheConfiguration(cacheTemplateName).persistence().stores().stream().filter((StoreConfiguration storeConfig) -> { - return storeConfig instanceof RemoteStoreConfiguration; + return storeConfig instanceof KeycloakRemoteStoreConfiguration; }).findFirst(); @@ -72,14 +72,20 @@ public class KeycloakRemoteStore extends RemoteStore { throw new CacheException("Unable to find remoteStore on cache '" + cacheTemplateName + "."); } - RemoteStoreConfiguration templateConfig = (RemoteStoreConfiguration) optional.get(); + KeycloakRemoteStoreConfiguration templateConfig = (KeycloakRemoteStoreConfiguration) optional.get(); - // We have template configuration, so create new configuration from it. Override just remoteCacheName + // We have template configuration, so create new configuration from it. Override just remoteCacheName and sessionsCache (not pretty, but works for now) PersistenceConfigurationBuilder readPersistenceBuilder = new ConfigurationBuilder().read(ctx.getCache().getCacheConfiguration()).persistence(); - RemoteStoreConfigurationBuilder configBuilder = new RemoteStoreConfigurationBuilder(readPersistenceBuilder); + KeycloakRemoteStoreConfigurationBuilder configBuilder = new KeycloakRemoteStoreConfigurationBuilder(readPersistenceBuilder); configBuilder.read(templateConfig); + // Rather log this to clearly show in the log that this might be a configuration mistake (Note that it can be expected for some cases) + if (!this.remoteCacheName.equals(ctx.getCache().getName())) { + logger.warnf("Cache name and remoteCache name are different - maybe it's expected. Cache name '%s', remoteCache name '%s'.", ctx.getCache().getName(), this.remoteCacheName); + } + configBuilder.remoteCacheName(this.remoteCacheName); + configBuilder.sessionCache(sessionCache); RemoteStoreConfiguration newCfg1 = configBuilder.create(); KeycloakRemoteStoreConfiguration newCfg = new KeycloakRemoteStoreConfiguration(newCfg1); @@ -93,6 +99,8 @@ public class KeycloakRemoteStore extends RemoteStore { logger.debugf("Skip overriding configuration from template for cache '%s'", ctx.getCache().getName()); } + logger.debugf("Using configuration for remote cache '%s': %s", remoteCacheName, getConfiguration().toString()); + super.start(); if (getRemoteCache() == null) { @@ -103,6 +111,10 @@ public class KeycloakRemoteStore extends RemoteStore { @Override public MarshalledEntry load(Object key) throws PersistenceException { + if (!getConfiguration().sessionCache()) { + return super.load(key); + } + logger.debugf("Calling load: '%s' for remote cache '%s'", key, remoteCacheName); MarshalledEntry entry = super.load(key); @@ -125,6 +137,11 @@ public class KeycloakRemoteStore extends RemoteStore { // Don't do anything. Iterate over remoteCache.keySet() can have big performance impact. We handle bulk load by ourselves if needed. @Override public void process(KeyFilter filter, CacheLoaderTask task, Executor executor, boolean fetchValue, boolean fetchMetadata) { + if (!getConfiguration().sessionCache()) { + super.process(filter, task, executor, fetchValue, fetchMetadata); + return; + } + logger.debugf("Skip calling process with filter '%s' on cache '%s'", filter, remoteCacheName); // super.process(filter, task, executor, fetchValue, fetchMetadata); } @@ -133,11 +150,19 @@ public class KeycloakRemoteStore extends RemoteStore { // Don't do anything. Writes handled by KC itself as we need more flexibility @Override public void write(MarshalledEntry entry) throws PersistenceException { + if (!getConfiguration().sessionCache()) { + super.write(entry); + return; + } } @Override public boolean delete(Object key) throws PersistenceException { + if (!getConfiguration().sessionCache()) { + return super.delete(key); + } + logger.debugf("Calling delete for key '%s' on cache '%s'", key, remoteCacheName); // Optimization - we don't need to know the previous value. diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfiguration.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfiguration.java index 79788e32dc..fa9b7db3d0 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfiguration.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfiguration.java @@ -31,18 +31,33 @@ import org.infinispan.persistence.remote.configuration.RemoteStoreConfiguration; public class KeycloakRemoteStoreConfiguration extends RemoteStoreConfiguration { static final AttributeDefinition USE_CONFIG_TEMPLATE_FROM_CACHE = AttributeDefinition.builder("useConfigTemplateFromCache", null, String.class).immutable().build(); + static final AttributeDefinition REMOTE_SERVERS = AttributeDefinition.builder("remoteServers", null, String.class).immutable().build(); + static final AttributeDefinition SESSION_CACHE = AttributeDefinition.builder("sessionCache", null, Boolean.class).immutable().build(); private final Attribute useConfigTemplateFromCache; + private final Attribute remoteServers; + private final Attribute sessionCache; public KeycloakRemoteStoreConfiguration(RemoteStoreConfiguration other) { super(other.attributes(), other.async(), other.singletonStore(), other.asyncExecutorFactory(), other.connectionPool()); useConfigTemplateFromCache = attributes.attribute(USE_CONFIG_TEMPLATE_FROM_CACHE.name()); + remoteServers = attributes.attribute(REMOTE_SERVERS.name()); + sessionCache = attributes.attribute(SESSION_CACHE.name()); } - public String useConfigTemplateFromCache() { return useConfigTemplateFromCache==null ? null : useConfigTemplateFromCache.get(); } + + + public String remoteServers() { + return remoteServers==null ? null : remoteServers.get(); + } + + + public Boolean sessionCache() { + return sessionCache==null ? false : sessionCache.get(); + } } diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfigurationBuilder.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfigurationBuilder.java index 2a0ec19705..a6ea1e6269 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfigurationBuilder.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakRemoteStoreConfigurationBuilder.java @@ -18,7 +18,9 @@ package org.keycloak.models.sessions.infinispan.remotestore; import java.lang.reflect.Field; +import java.util.List; import java.util.Map; +import java.util.StringTokenizer; import org.infinispan.commons.CacheConfigurationException; import org.infinispan.commons.configuration.attributes.Attribute; @@ -39,13 +41,22 @@ public class KeycloakRemoteStoreConfigurationBuilder extends RemoteStoreConfigur // No better way to add new attribute definition to superclass :/ try { - AttributeDefinition def = KeycloakRemoteStoreConfiguration.USE_CONFIG_TEMPLATE_FROM_CACHE; - Attribute attribute = def.toAttribute(); - Field f = Reflections.findDeclaredField(AttributeSet.class, "attributes"); f.setAccessible(true); Map> attributesInternal = (Map>) f.get(this.attributes); + + AttributeDefinition def = KeycloakRemoteStoreConfiguration.USE_CONFIG_TEMPLATE_FROM_CACHE; + Attribute attribute = def.toAttribute(); attributesInternal.put(def.name(), attribute); + + def = KeycloakRemoteStoreConfiguration.REMOTE_SERVERS; + attribute = def.toAttribute(); + attributesInternal.put(def.name(), attribute); + + AttributeDefinition defBool = KeycloakRemoteStoreConfiguration.SESSION_CACHE; + Attribute attributeBool = defBool.toAttribute(); + attributesInternal.put(defBool.name(), attributeBool); + } catch (IllegalAccessException iae) { throw new CacheConfigurationException(iae); } @@ -54,8 +65,58 @@ public class KeycloakRemoteStoreConfigurationBuilder extends RemoteStoreConfigur @Override public KeycloakRemoteStoreConfiguration create() { + String remoteServersAttr = attributes.attribute(KeycloakRemoteStoreConfiguration.REMOTE_SERVERS).get(); + boolean isServersAlreadySet = isServersAlreadySet(); + if (remoteServersAttr != null && !isServersAlreadySet) { + parseRemoteServersAttr(remoteServersAttr); + } + RemoteStoreConfiguration cfg = super.create(); KeycloakRemoteStoreConfiguration cfg2 = new KeycloakRemoteStoreConfiguration(cfg); return cfg2; } + + + public KeycloakRemoteStoreConfigurationBuilder useConfigTemplateFromCache(String useConfigTemplateFromCache) { + attributes.attribute(KeycloakRemoteStoreConfiguration.USE_CONFIG_TEMPLATE_FROM_CACHE).set(useConfigTemplateFromCache); + return this; + } + + + public KeycloakRemoteStoreConfigurationBuilder remoteServers(String remoteServers) { + attributes.attribute(KeycloakRemoteStoreConfiguration.REMOTE_SERVERS).set(remoteServers); + return this; + } + + + public KeycloakRemoteStoreConfigurationBuilder sessionCache(Boolean sessionCache) { + attributes.attribute(KeycloakRemoteStoreConfiguration.SESSION_CACHE).set(sessionCache); + return this; + } + + + private void parseRemoteServersAttr(String remoteServers) { + StringTokenizer st = new StringTokenizer(remoteServers, ","); + + while (st.hasMoreElements()) { + String nodeStr = st.nextToken(); + String[] node = nodeStr.trim().split(":", 2); + + addServer() + .host(node[0].trim()) + .port(Integer.parseInt(node[1].trim())); + } + } + + + private boolean isServersAlreadySet() { + try { + Field f = Reflections.findDeclaredField(RemoteStoreConfigurationBuilder.class, "servers"); + f.setAccessible(true); + List originalRemoteServers = (List) f.get(this); + return !originalRemoteServers.isEmpty(); + } catch (IllegalAccessException iae) { + throw new RuntimeException(iae); + } + } } diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakTcpTransportFactory.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakTcpTransportFactory.java new file mode 100644 index 0000000000..a8985f248d --- /dev/null +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/KeycloakTcpTransportFactory.java @@ -0,0 +1,120 @@ +/* + * Copyright 2017 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.models.sessions.infinispan.remotestore; + +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.infinispan.client.hotrod.configuration.Configuration; +import org.infinispan.client.hotrod.configuration.ServerConfiguration; +import org.infinispan.client.hotrod.event.ClientListenerNotifier; +import org.infinispan.client.hotrod.impl.protocol.Codec; +import org.infinispan.client.hotrod.impl.transport.tcp.TcpTransportFactory; +import org.jboss.logging.Logger; +import org.keycloak.common.util.reflections.Reflections; + +/** + * @author Marek Posolda + */ +public class KeycloakTcpTransportFactory extends TcpTransportFactory { + + protected static final Logger logger = Logger.getLogger(KeycloakTcpTransportFactory.class); + + private Collection kcInitialServers; + + @Override + public void start(Codec codec, Configuration configuration, AtomicInteger defaultCacheTopologyId, ClientListenerNotifier listenerNotifier) { + kcInitialServers = new HashSet<>(); + + for (ServerConfiguration server : configuration.servers()) { + InetSocketAddress hostnameAddress = new InetSocketAddress(server.host(), server.port()); + kcInitialServers.add(hostnameAddress); + + // Retrieve servers by IP addresses too, as we need to compare by IP addresses + try { + String ip = InetAddress.getByName(server.host()).getHostAddress(); + InetSocketAddress ipAddress = new InetSocketAddress(ip, server.port()); + kcInitialServers.add(ipAddress); + + InetSocketAddress unresolved = InetSocketAddress.createUnresolved(ip, server.port()); + kcInitialServers.add(unresolved); + } catch (UnknownHostException uhe) { + logger.warnf(uhe, "Wasn't able to retrieve IP address for host '%s'", server.host()); + } + + } + + logger.debugf("Keycloak initial servers: %s", kcInitialServers); + + super.start(codec, configuration, defaultCacheTopologyId, listenerNotifier); + } + + + @Override + public void updateServers(Collection newServers, byte[] cacheName, boolean quiet) { + try { + logger.debugf("Update servers called: %s, cacheName: %s", newServers, new String(cacheName, "UTF-8")); + + Collection filteredServers = getFilteredNewServers(newServers); + + logger.debugf("Update servers after filter: %s, cacheName: %s", filteredServers, new String(cacheName, "UTF-8")); + + super.updateServers(filteredServers, cacheName, quiet); + + } catch (UnsupportedEncodingException uee) { + throw new RuntimeException(uee); + } + } + + + // Return just those servers, which are part of the originally configured "kcInitialServers". + // Assume that the other JDG servers are part of same cluster, but are in different DC. Hence don't include them in the topology view + private Collection getFilteredNewServers(Collection newServers) { + Collection initialServers = getInitialServers(); + Collection filteredServers = newServers.stream().filter((SocketAddress newAddress) -> { + + boolean presentInInitialServers = initialServers.contains(newAddress); + + if (!presentInInitialServers) { + logger.debugf("Server'%s' not present in initial servers. Probably server from different DC. Will filter it from the view", newAddress); + } + + return presentInInitialServers; + + }).collect(Collectors.toList()); + + return filteredServers; + } + + + protected Collection getInitialServers() { + return kcInitialServers; + } + + + +} diff --git a/testsuite/integration-arquillian/tests/base/src/test/resources/log4j.properties b/testsuite/integration-arquillian/tests/base/src/test/resources/log4j.properties index 3dcf46ac7c..d4b76b6de2 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/resources/log4j.properties +++ b/testsuite/integration-arquillian/tests/base/src/test/resources/log4j.properties @@ -59,6 +59,8 @@ log4j.logger.org.keycloak.keys.infinispan=${keycloak.infinispan.logging.level} log4j.logger.org.keycloak.models.cache.infinispan=${keycloak.infinispan.logging.level} log4j.logger.org.keycloak.models.sessions.infinispan=${keycloak.infinispan.logging.level} +log4j.logger.org.infinispan.client.hotrod.impl=info + # Enable to view kerberos/spnego logging # log4j.logger.org.keycloak.broker.kerberos=trace diff --git a/testsuite/utils/src/main/resources/META-INF/keycloak-server.json b/testsuite/utils/src/main/resources/META-INF/keycloak-server.json index a8aa46dcdf..12ffb33020 100755 --- a/testsuite/utils/src/main/resources/META-INF/keycloak-server.json +++ b/testsuite/utils/src/main/resources/META-INF/keycloak-server.json @@ -93,7 +93,7 @@ "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}", "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}", "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}", - "remoteStoreHost": "${keycloak.connectionsjen neInfinispan.remoteStoreHost:localhost}", + "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}", "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}" } },