diff --git a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java index f9a30703ac..916db65cc3 100755 --- a/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java @@ -164,9 +164,16 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon throw new RuntimeException("Invalid value for sessionsMode"); } - sessionConfigBuilder.clustering().hash() - .numOwners(config.getInt("sessionsOwners", 2)) - .numSegments(config.getInt("sessionsSegments", 60)).build(); + int l1Lifespan = config.getInt("l1Lifespan", 600000); + boolean l1Enabled = l1Lifespan > 0; + sessionConfigBuilder.clustering() + .hash() + .numOwners(config.getInt("sessionsOwners", 2)) + .numSegments(config.getInt("sessionsSegments", 60)) + .l1() + .enabled(l1Enabled) + .lifespan(l1Lifespan) + .build(); } Configuration sessionCacheConfiguration = sessionConfigBuilder.build(); diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java index c21f787098..7d68c18ae4 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java @@ -19,6 +19,7 @@ package org.keycloak.models.sessions.infinispan; import org.infinispan.Cache; import org.infinispan.CacheStream; +import org.infinispan.context.Flag; import org.jboss.logging.Logger; import org.keycloak.common.util.Time; import org.keycloak.models.ClientInitialAccessModel; @@ -291,6 +292,7 @@ public class InfinispanUserSessionProvider implements UserSessionProvider { @Override public void removeExpired(RealmModel realm) { + log.debugf("Removing expired sessions"); removeExpiredUserSessions(realm); removeExpiredClientSessions(realm); removeExpiredOfflineUserSessions(realm); @@ -302,9 +304,13 @@ public class InfinispanUserSessionProvider implements UserSessionProvider { int expired = Time.currentTime() - realm.getSsoSessionMaxLifespan(); int expiredRefresh = Time.currentTime() - realm.getSsoSessionIdleTimeout(); - Iterator> itr = sessionCache.entrySet().stream().filter(UserSessionPredicate.create(realm.getId()).expired(expired, expiredRefresh)).iterator(); + // Each cluster node cleanups just local sessions, which are those owned by himself (+ few more taking l1 cache into account) + Iterator> itr = sessionCache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) + .entrySet().stream().filter(UserSessionPredicate.create(realm.getId()).expired(expired, expiredRefresh)).iterator(); + int counter = 0; while (itr.hasNext()) { + counter++; UserSessionEntity entity = (UserSessionEntity) itr.next().getValue(); tx.remove(sessionCache, entity.getId()); @@ -314,23 +320,38 @@ public class InfinispanUserSessionProvider implements UserSessionProvider { } } } + + log.debugf("Removed %d expired user sessions for realm '%s'", counter, realm.getName()); } private void removeExpiredClientSessions(RealmModel realm) { int expiredDettachedClientSession = Time.currentTime() - RealmInfoUtil.getDettachedClientSessionLifespan(realm); - Iterator> itr = sessionCache.entrySet().stream().filter(ClientSessionPredicate.create(realm.getId()).expiredRefresh(expiredDettachedClientSession).requireNullUserSession()).iterator(); + // Each cluster node cleanups just local sessions, which are those owned by himself (+ few more taking l1 cache into account) + Iterator> itr = sessionCache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) + .entrySet().stream().filter(ClientSessionPredicate.create(realm.getId()).expiredRefresh(expiredDettachedClientSession).requireNullUserSession()).iterator(); + + int counter = 0; while (itr.hasNext()) { + counter++; tx.remove(sessionCache, itr.next().getKey()); } + + log.debugf("Removed %d expired client sessions for realm '%s'", counter, realm.getName()); } private void removeExpiredOfflineUserSessions(RealmModel realm) { UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class); int expiredOffline = Time.currentTime() - realm.getOfflineSessionIdleTimeout(); - Iterator> itr = offlineSessionCache.entrySet().stream().filter(UserSessionPredicate.create(realm.getId()).expired(null, expiredOffline)).iterator(); + // Each cluster node cleanups just local sessions, which are those owned by himself (+ few more taking l1 cache into account) + UserSessionPredicate predicate = UserSessionPredicate.create(realm.getId()).expired(null, expiredOffline); + Iterator> itr = offlineSessionCache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) + .entrySet().stream().filter(predicate).iterator(); + + int counter = 0; while (itr.hasNext()) { + counter++; UserSessionEntity entity = (UserSessionEntity) itr.next().getValue(); tx.remove(offlineSessionCache, entity.getId()); @@ -340,22 +361,32 @@ public class InfinispanUserSessionProvider implements UserSessionProvider { tx.remove(offlineSessionCache, clientSessionId); } } + + log.debugf("Removed %d expired offline user sessions for realm '%s'", counter, realm.getName()); } private void removeExpiredOfflineClientSessions(RealmModel realm) { UserSessionPersisterProvider persister = session.getProvider(UserSessionPersisterProvider.class); int expiredOffline = Time.currentTime() - realm.getOfflineSessionIdleTimeout(); - Iterator itr = offlineSessionCache.entrySet().stream().filter(ClientSessionPredicate.create(realm.getId()).expiredRefresh(expiredOffline)).map(Mappers.sessionId()).iterator(); + // Each cluster node cleanups just local sessions, which are those owned by himself (+ few more taking l1 cache into account) + Iterator itr = offlineSessionCache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) + .entrySet().stream().filter(ClientSessionPredicate.create(realm.getId()).expiredRefresh(expiredOffline)).map(Mappers.sessionId()).iterator(); + + int counter = 0; while (itr.hasNext()) { + counter++; String sessionId = itr.next(); tx.remove(offlineSessionCache, sessionId); persister.removeClientSession(sessionId, true); } + + log.debugf("Removed %d expired offline client sessions for realm '%s'", counter, realm.getName()); } private void removeExpiredClientInitialAccess(RealmModel realm) { - Iterator itr = sessionCache.entrySet().stream().filter(ClientInitialAccessPredicate.create(realm.getId()).expired(Time.currentTime())).map(Mappers.sessionId()).iterator(); + Iterator itr = sessionCache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) + .entrySet().stream().filter(ClientInitialAccessPredicate.create(realm.getId()).expired(Time.currentTime())).map(Mappers.sessionId()).iterator(); while (itr.hasNext()) { tx.remove(sessionCache, itr.next()); } diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java index d636ae379e..4b04d9b752 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/SessionInitializerWorker.java @@ -60,7 +60,7 @@ public class SessionInitializerWorker implements DistributedCallableMarek Posolda + */ +@Ignore +public class ClusterSessionCleanerTest { + + protected static final Logger logger = Logger.getLogger(ClusterSessionCleanerTest.class); + + private static final String REALM_NAME = "test"; + + @ClassRule + public static KeycloakRule server1 = new KeycloakRule(); + + @ClassRule + public static KeycloakRule server2 = new KeycloakRule() { + + @Override + protected void configureServer(KeycloakServer server) { + server.getConfig().setPort(8082); + } + + @Override + protected void importRealm() { + } + + @Override + protected void removeTestRealms() { + } + + }; + + @Test + public void testClusterPeriodicSessionCleanups() throws Exception { + // Add some userSessions on server1 + KeycloakSession session1 = server1.startSession(); + RealmModel realm1 = session1.realms().getRealmByName(REALM_NAME); + UserModel user1 = session1.users().getUserByUsername("test-user@localhost", realm1); + for (int i=0 ; i<15 ; i++) { + session1.sessions().createUserSession(realm1, user1, user1.getUsername(), "127.0.0.1", "form", true, null, null); + } + session1 = commit(server1, session1); + + // Add some userSessions on server2 + KeycloakSession session2 = server2.startSession(); + RealmModel realm2 = session2.realms().getRealmByName(REALM_NAME); + UserModel user2 = session2.users().getUserByUsername("test-user@localhost", realm2); + // Check we are really in cluster (same user ids) + Assert.assertEquals(user2.getId(), user1.getId()); + + for (int i=0 ; i<15 ; i++) { + session2.sessions().createUserSession(realm2, user2, user2.getUsername(), "127.0.0.1", "form", true, null, null); + } + session2 = commit(server2, session2); + + // Assert sessions on both nodes + List sessions1 = getSessions(session1); + List sessions2 = getSessions(session2); + Assert.assertEquals(30, sessions1.size()); + Assert.assertEquals(30, sessions2.size()); + logger.info("Before offset: sessions1 : " + sessions1.size()); + logger.info("Before offset: sessions2 : " + sessions2.size()); + + + // set Time offset and run periodic cleaner on server1 + Time.setOffset(999999); + realm1 = session1.realms().getRealmByName(REALM_NAME); + session1.sessions().removeExpired(realm1); + session1 = commit(server1, session1); + + // Ensure some sessions still there + sessions1 = getSessions(session1); + sessions2 = getSessions(session2); + logger.info("After server1 periodic clean: sessions1 : " + sessions1.size()); + logger.info("After server1 periodic clean: sessions2 : " + sessions2.size()); + + + // Run periodic cleaner on server2 + realm2 = session2.realms().getRealmByName(REALM_NAME); + session2.sessions().removeExpired(realm2); + session2 = commit(server1, session2); + + // Ensure there are no sessions on server1 or server2 + sessions1 = getSessions(session1); + sessions2 = getSessions(session2); + Assert.assertTrue(sessions1.isEmpty()); + Assert.assertTrue(sessions2.isEmpty()); + logger.info("After both periodic cleans: sessions1 : " + sessions1.size()); + logger.info("After both periodic cleans: sessions2 : " + sessions2.size()); + } + + private List getSessions(KeycloakSession session) { + RealmModel realm = session.realms().getRealmByName(REALM_NAME); + UserModel user = session.users().getUserByUsername("test-user@localhost", realm); + return session.sessions().getUserSessions(realm, user); + } + + private KeycloakSession commit(KeycloakRule rule, KeycloakSession session) throws Exception { + session.getTransactionManager().commit(); + session.close(); + return rule.startSession(); + } + +} diff --git a/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractOfflineCacheCommand.java b/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractOfflineCacheCommand.java index 59be8aadee..f94cea05d6 100644 --- a/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractOfflineCacheCommand.java +++ b/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/AbstractOfflineCacheCommand.java @@ -223,4 +223,18 @@ public abstract class AbstractOfflineCacheCommand extends AbstractCommand { } } + + public static class SizeLocalCommand extends AbstractOfflineCacheCommand { + + @Override + public String getName() { + return "sizeLocal"; + } + + @Override + protected void doRunCacheCommand(KeycloakSession session, Cache cache) { + log.info("Size local: " + cache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL).size()); + } + } + } diff --git a/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java b/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java index 9b2c17aaca..b1ff087950 100644 --- a/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java +++ b/testsuite/integration/src/test/java/org/keycloak/testsuite/util/cli/TestsuiteCLI.java @@ -47,6 +47,7 @@ public class TestsuiteCLI { AbstractOfflineCacheCommand.GetCommand.class, AbstractOfflineCacheCommand.GetMultipleCommand.class, AbstractOfflineCacheCommand.GetLocalCommand.class, + AbstractOfflineCacheCommand.SizeLocalCommand.class, AbstractOfflineCacheCommand.RemoveCommand.class, AbstractOfflineCacheCommand.SizeCommand.class, AbstractOfflineCacheCommand.ListCommand.class, diff --git a/testsuite/integration/src/test/resources/META-INF/keycloak-server.json b/testsuite/integration/src/test/resources/META-INF/keycloak-server.json index 27e9f5ec9e..40a15e98e6 100755 --- a/testsuite/integration/src/test/resources/META-INF/keycloak-server.json +++ b/testsuite/integration/src/test/resources/META-INF/keycloak-server.json @@ -97,7 +97,8 @@ "default": { "clustered": "${keycloak.connectionsInfinispan.clustered:false}", "async": "${keycloak.connectionsInfinispan.async:false}", - "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:2}", + "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}", + "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}", "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}", "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}", "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"