KEYCLOAK-5030 Change action tokens cache type to distributed

This commit is contained in:
Hynek Mlnarik 2017-06-26 09:55:21 +02:00
parent 58e5f41415
commit 955cbc76d7
13 changed files with 84 additions and 57 deletions

View file

@ -206,13 +206,13 @@ if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache-
echo echo
end-if end-if
if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:read-resource if (outcome == failed) of /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:read-resource
echo Adding local-cache=actionTokens to keycloak cache container... echo Adding distributed-cache=actionTokens to keycloak cache container...
/profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:add(indexing=NONE,start=LAZY) /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:add(indexing=NONE,mode=SYNC,owners=2)
/profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE)
/profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1)
/profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000)
/profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) /profile=$clusteredProfile/subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1)
echo echo
end-if end-if

View file

@ -211,13 +211,13 @@ if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/distrib
echo echo
end-if end-if
if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:read-resource if (outcome == failed) of /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:read-resource
echo Adding local-cache=actionTokens to keycloak cache container... echo Adding distributed-cache=actionTokens to keycloak cache container...
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/:add(indexing=NONE,start=LAZY) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/:add(indexing=NONE,mode=SYNC,owners=2)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=strategy,value=NONE)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=eviction/:write-attribute(name=max-entries,value=-1)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=interval,value=300000)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/component=expiration/:write-attribute(name=max-idle,value=-1)
echo echo
end-if end-if

View file

@ -16,7 +16,7 @@ embed-server --server-config=standalone-ha.xml
/subsystem=infinispan/cache-container=keycloak/local-cache=keys:add() /subsystem=infinispan/cache-container=keycloak/local-cache=keys:add()
/subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:add(max-entries=1000,strategy=LRU) /subsystem=infinispan/cache-container=keycloak/local-cache=keys/eviction=EVICTION:add(max-entries=1000,strategy=LRU)
/subsystem=infinispan/cache-container=keycloak/local-cache=keys/expiration=EXPIRATION:add(max-idle=3600000) /subsystem=infinispan/cache-container=keycloak/local-cache=keys/expiration=EXPIRATION:add(max-idle=3600000)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens:add() /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens:add(indexing="NONE",mode="SYNC",owners="2")
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/eviction=EVICTION:add(max-entries=-1,strategy=NONE)
/subsystem=infinispan/cache-container=keycloak/local-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000) /subsystem=infinispan/cache-container=keycloak/distributed-cache=actionTokens/expiration=EXPIRATION:add(max-idle=-1,interval=300000)
/extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem) /extension=org.keycloak.keycloak-server-subsystem/:add(module=org.keycloak.keycloak-server-subsystem)

View file

@ -248,7 +248,14 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
cacheManager.defineConfiguration(InfinispanConnectionProvider.KEYS_CACHE_NAME, getKeysCacheConfig()); cacheManager.defineConfiguration(InfinispanConnectionProvider.KEYS_CACHE_NAME, getKeysCacheConfig());
cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true); cacheManager.getCache(InfinispanConnectionProvider.KEYS_CACHE_NAME, true);
cacheManager.defineConfiguration(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, getActionTokenCacheConfig()); final ConfigurationBuilder actionTokenCacheConfigBuilder = getActionTokenCacheConfig();
if (clustered) {
actionTokenCacheConfigBuilder.clustering().cacheMode(async ? CacheMode.REPL_ASYNC : CacheMode.REPL_SYNC);
}
if (jdgEnabled) {
configureRemoteActionTokenCacheStore(actionTokenCacheConfigBuilder, async);
}
cacheManager.defineConfiguration(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, actionTokenCacheConfigBuilder.build());
cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true); cacheManager.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE, true);
long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().eviction().maxEntries(); long authzRevisionsMaxEntries = cacheManager.getCache(InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME).getCacheConfiguration().eviction().maxEntries();
@ -301,6 +308,30 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
} }
private void configureRemoteActionTokenCacheStore(ConfigurationBuilder builder, boolean async) {
String jdgServer = config.get("remoteStoreServer", "localhost");
Integer jdgPort = config.getInt("remoteStorePort", 11222);
builder.persistence()
.passivation(false)
.addStore(RemoteStoreConfigurationBuilder.class)
.fetchPersistentState(false)
.ignoreModifications(false)
.purgeOnStartup(false)
.preload(true)
.shared(true)
.remoteCacheName(InfinispanConnectionProvider.ACTION_TOKEN_CACHE)
.rawValues(true)
.forceReturnValues(false)
.marshaller(KeycloakHotRodMarshallerFactory.class.getName())
.addServer()
.host(jdgServer)
.port(jdgPort)
.async()
.enabled(async);
}
protected Configuration getKeysCacheConfig() { protected Configuration getKeysCacheConfig() {
ConfigurationBuilder cb = new ConfigurationBuilder(); ConfigurationBuilder cb = new ConfigurationBuilder();
cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX); cb.eviction().strategy(EvictionStrategy.LRU).type(EvictionType.COUNT).size(InfinispanConnectionProvider.KEYS_CACHE_DEFAULT_MAX);
@ -308,7 +339,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
return cb.build(); return cb.build();
} }
private Configuration getActionTokenCacheConfig() { private ConfigurationBuilder getActionTokenCacheConfig() {
ConfigurationBuilder cb = new ConfigurationBuilder(); ConfigurationBuilder cb = new ConfigurationBuilder();
cb.eviction() cb.eviction()
@ -319,7 +350,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
.maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS) .maxIdle(InfinispanConnectionProvider.ACTION_TOKEN_MAX_IDLE_SECONDS, TimeUnit.SECONDS)
.wakeUpInterval(InfinispanConnectionProvider.ACTION_TOKEN_WAKE_UP_INTERVAL_SECONDS, TimeUnit.SECONDS); .wakeUpInterval(InfinispanConnectionProvider.ACTION_TOKEN_WAKE_UP_INTERVAL_SECONDS, TimeUnit.SECONDS);
return cb.build(); return cb;
} }
private static final Object CHANNEL_INIT_SYNCHRONIZER = new Object(); private static final Object CHANNEL_INIT_SYNCHRONIZER = new Object();

View file

@ -17,13 +17,14 @@
package org.keycloak.models.sessions.infinispan; package org.keycloak.models.sessions.infinispan;
import org.keycloak.cluster.ClusterProvider; import org.keycloak.cluster.ClusterProvider;
import org.keycloak.common.util.Time;
import org.keycloak.models.*; import org.keycloak.models.*;
import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent; import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity; import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey; import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
import java.util.*; import java.util.*;
import java.util.concurrent.TimeUnit;
import org.infinispan.Cache; import org.infinispan.Cache;
/** /**
@ -57,9 +58,7 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
ActionTokenReducedKey tokenKey = new ActionTokenReducedKey(key.getUserId(), key.getActionId(), key.getActionVerificationNonce()); ActionTokenReducedKey tokenKey = new ActionTokenReducedKey(key.getUserId(), key.getActionId(), key.getActionVerificationNonce());
ActionTokenValueEntity tokenValue = new ActionTokenValueEntity(notes); ActionTokenValueEntity tokenValue = new ActionTokenValueEntity(notes);
ClusterProvider cluster = session.getProvider(ClusterProvider.class); this.tx.put(actionKeyCache, tokenKey, tokenValue, key.getExpiration() - Time.currentTime(), TimeUnit.SECONDS);
AddInvalidatedActionTokenEvent event = new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue);
this.tx.notify(cluster, generateActionTokenEventId(), event, false);
} }
private static String generateActionTokenEventId() { private static String generateActionTokenEventId() {
@ -92,6 +91,7 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
return value; return value;
} }
@Override
public void removeAll(String userId, String actionId) { public void removeAll(String userId, String actionId) {
if (userId == null || actionId == null) { if (userId == null || actionId == null) {
return; return;

View file

@ -19,16 +19,16 @@ package org.keycloak.models.sessions.infinispan;
import org.keycloak.Config; import org.keycloak.Config;
import org.keycloak.Config.Scope; import org.keycloak.Config.Scope;
import org.keycloak.cluster.ClusterProvider; import org.keycloak.cluster.ClusterProvider;
import org.keycloak.common.util.Time;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider; import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.models.*; import org.keycloak.models.*;
import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent; import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity; import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey; import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.TimeUnit; import java.util.stream.Collectors;
import org.infinispan.AdvancedCache;
import org.infinispan.Cache; import org.infinispan.Cache;
import org.infinispan.context.Flag; import org.infinispan.context.Flag;
import org.infinispan.remoting.transport.Address; import org.infinispan.remoting.transport.Address;
@ -76,22 +76,17 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto
LOG.debugf("[%s] Removing token invalidation for user+action: userId=%s, actionId=%s", cacheAddress, e.getUserId(), e.getActionId()); LOG.debugf("[%s] Removing token invalidation for user+action: userId=%s, actionId=%s", cacheAddress, e.getUserId(), e.getActionId());
cache AdvancedCache<ActionTokenReducedKey, ActionTokenValueEntity> localCache = cache
.getAdvancedCache() .getAdvancedCache()
.withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD) .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD);
List<ActionTokenReducedKey> toRemove = localCache
.keySet() .keySet()
.stream() .stream()
.filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId())) .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId()))
.forEach(cache::remove); .collect(Collectors.toList());
} else if (event instanceof AddInvalidatedActionTokenEvent) {
AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event;
LOG.debugf("[%s] Invalidating token %s", cacheAddress, e.getKey()); toRemove.forEach(localCache::remove);
if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) {
cache.put(e.getKey(), e.getTokenValue());
} else {
cache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS);
}
} }
}); });

View file

@ -30,6 +30,7 @@
<xsl:copy> <xsl:copy>
<xsl:apply-templates select="@* | node()" /> <xsl:apply-templates select="@* | node()" />
<local-cache name="work" start="EAGER" batching="false" /> <local-cache name="work" start="EAGER" batching="false" />
<local-cache name="actionTokens" start="EAGER" batching="false" />
</xsl:copy> </xsl:copy>
</xsl:template> </xsl:template>
@ -38,6 +39,7 @@
<xsl:copy> <xsl:copy>
<xsl:apply-templates select="@* | node()" /> <xsl:apply-templates select="@* | node()" />
<replicated-cache name="work" start="EAGER" batching="false" /> <replicated-cache name="work" start="EAGER" batching="false" />
<replicated-cache name="actionTokens" start="EAGER" batching="false" />
</xsl:copy> </xsl:copy>
</xsl:template> </xsl:template>

View file

@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.function.BiConsumer; import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import org.hamcrest.Matcher; import org.hamcrest.Matcher;
import org.junit.Before; import org.junit.Before;

View file

@ -116,11 +116,11 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
* @return * @return
*/ */
protected Keycloak getAdminClientFor(ContainerInfo node) { protected Keycloak getAdminClientFor(ContainerInfo node) {
Keycloak adminClient = backendAdminClients.get(node); Keycloak client = backendAdminClients.get(node);
if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) { if (client == null && node.equals(suiteContext.getAuthServerInfo())) {
adminClient = this.adminClient; client = this.adminClient;
} }
return adminClient; return client;
} }
/** /**

View file

@ -34,7 +34,6 @@ import javax.mail.internet.MimeMessage;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.jboss.arquillian.graphene.page.Page; import org.jboss.arquillian.graphene.page.Page;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -45,7 +44,6 @@ import org.keycloak.testsuite.arquillian.InfinispanStatistics.Constants;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.hamcrest.Matchers; import org.hamcrest.Matchers;
import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
/** /**
@ -117,21 +115,23 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
old -> greaterThan((Comparable) 0l) old -> greaterThan((Comparable) 0l)
); );
// Verify that the caches are synchronized
assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES),
is(cacheDc1Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES)));
assertEquals("Your account has been updated.", driver.getTitle()); assertEquals("Your account has been updated.", driver.getTitle());
// Verify that there was an action token added in the node which was targetted by the link
assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
disableDcOnLoadBalancer(0); disableDcOnLoadBalancer(0);
enableDcOnLoadBalancer(1); enableDcOnLoadBalancer(1);
driver.navigate().to(link); // Make sure that after going to the link, the invalidated action token has been retrieved from Infinispan server cluster in the other DC
assertSingleStatistics(cacheDc1Node0Statistics, Constants.STAT_CACHE_NUMBER_OF_ENTRIES,
() -> driver.navigate().to(link),
Matchers::greaterThan
);
errorPage.assertCurrent(); errorPage.assertCurrent();
} }
@Ignore("KEYCLOAK-5030")
@Test @Test
public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException { public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
disableDcOnLoadBalancer(1); disableDcOnLoadBalancer(1);

View file

@ -224,7 +224,7 @@
<property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property> <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
<property name="bindAddress">localhost</property> <property name="bindAddress">localhost</property>
<property name="bindHttpPort">${auth.server.http.port}</property> <property name="bindHttpPort">${auth.server.http.port}</property>
<property name="nodes">auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112</property> <property name="nodes">auth-server-undertow-cross-dc-0_1=http://localhost:8101,auth-server-undertow-cross-dc-0_2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1_1=http://localhost:8111,auth-server-undertow-cross-dc-1_2-manual=http://localhost:8112</property>
</configuration> </configuration>
</container> </container>

View file

@ -113,10 +113,10 @@
<eviction max-entries="1000" strategy="LRU"/> <eviction max-entries="1000" strategy="LRU"/>
<expiration max-idle="3600000" /> <expiration max-idle="3600000" />
</local-cache> </local-cache>
<local-cache name="actionTokens"> <distributed-cache name="actionTokens" mode="SYNC" owners="2">
<eviction max-entries="-1" strategy="NONE"/> <eviction max-entries="-1" strategy="NONE"/>
<expiration max-idle="-1" interval="300000"/> <expiration max-idle="-1" interval="300000"/>
</local-cache> </distributed-cache>
</cache-container> </cache-container>
<cache-container name="server" aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server"> <cache-container name="server" aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server">
<transport lock-timeout="60000"/> <transport lock-timeout="60000"/>

View file

@ -116,10 +116,10 @@
<eviction max-entries="1000" strategy="LRU"/> <eviction max-entries="1000" strategy="LRU"/>
<expiration max-idle="3600000" /> <expiration max-idle="3600000" />
</local-cache> </local-cache>
<local-cache name="actionTokens"> <distributed-cache name="actionTokens" mode="SYNC" owners="2">
<eviction max-entries="-1" strategy="NONE"/> <eviction max-entries="-1" strategy="NONE"/>
<expiration max-idle="-1" interval="300000"/> <expiration max-idle="-1" interval="300000"/>
</local-cache> </distributed-cache>
</cache-container> </cache-container>
<cache-container name="server" aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server"> <cache-container name="server" aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server">
<transport lock-timeout="60000"/> <transport lock-timeout="60000"/>