libre.sh/keycloak-scim
3
0
Fork 0
Code Issues 15 Pull requests Releases 1 Packages Activity Actions

Remove Infinispan workarounds introduced to prevent deadlocks (#21862)

Browse source 
This should no longer be necessary after the upgrade to Infinispan 14.0.13.Final and ISPN-13666 being resolved.

Closes #9871
This commit is contained in:
Alexander Schwartz 2023-07-24 09:50:32 +02:00 committed by GitHub
parent 05b8b9ee51
commit bd0f87fc4d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java
View file

@ -33,7 +33,6 @@ import org.keycloak.cluster.ClusterProviderFactory;
import org.keycloak.common.Profile; import org.keycloak.common.Profile;
import org.keycloak.common.util.Retry; import org.keycloak.common.util.Retry;
import org.keycloak.common.util.Time; import org.keycloak.common.util.Time;
import org.keycloak.connections.infinispan.DefaultInfinispanConnectionProviderFactory;
import org.keycloak.connections.infinispan.InfinispanConnectionProvider; import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
import org.keycloak.connections.infinispan.TopologyInfo; import org.keycloak.connections.infinispan.TopologyInfo;
import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSession;
@ -221,17 +220,7 @@ public class InfinispanClusterProviderFactory implements ClusterProviderFactory,
} }
logger.debugf("Nodes %s removed from cluster. Removing tasks locked by this nodes", removedNodesAddresses.toString()); logger.debugf("Nodes %s removed from cluster. Removing tasks locked by this nodes", removedNodesAddresses.toString());
/* workCache.entrySet().removeIf(new LockEntryPredicate(removedNodesAddresses));
workaround for Infinispan 12.1.7.Final to prevent a deadlock while
DefaultInfinispanConnectionProviderFactory is shutting down PersistenceManagerImpl
that acquires a writeLock and this removal that acquires a readLock.
First seen with https://issues.redhat.com/browse/ISPN-13664 and still occurs probably due to
https://issues.redhat.com/browse/ISPN-13666 in 13.0.10
Tracked in https://github.com/keycloak/keycloak/issues/9871
*/
synchronized (DefaultInfinispanConnectionProviderFactory.class) {
workCache.entrySet().removeIf(new LockEntryPredicate(removedNodesAddresses));
}
} }
} catch (Throwable t) { } catch (Throwable t) {
logger.error("caught exception in ViewChangeListener", t); logger.error("caught exception in ViewChangeListener", t);

59
model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanNotificationsManager.java
View file

@ -51,7 +51,6 @@ import org.keycloak.cluster.ClusterListener;
import org.keycloak.cluster.ClusterProvider; import org.keycloak.cluster.ClusterProvider;
import org.keycloak.common.util.ConcurrentMultivaluedHashMap; import org.keycloak.common.util.ConcurrentMultivaluedHashMap;
import org.keycloak.common.util.Retry; import org.keycloak.common.util.Retry;
import org.keycloak.connections.infinispan.DefaultInfinispanConnectionProviderFactory;
import org.keycloak.executors.ExecutorsProvider; import org.keycloak.executors.ExecutorsProvider;
import org.keycloak.models.KeycloakSession; import org.keycloak.models.KeycloakSession;
import org.infinispan.client.hotrod.exceptions.HotRodClientException; import org.infinispan.client.hotrod.exceptions.HotRodClientException;
@ -71,7 +70,7 @@ public class InfinispanNotificationsManager {
private final Cache<String, Serializable> workCache; private final Cache<String, Serializable> workCache;
private final RemoteCache workRemoteCache; private final RemoteCache<String, Serializable> workRemoteCache;
private final String myAddress; private final String myAddress;
@ -80,7 +79,7 @@ public class InfinispanNotificationsManager {
private final ExecutorService listenersExecutor; private final ExecutorService listenersExecutor;
protected InfinispanNotificationsManager(Cache<String, Serializable> workCache, RemoteCache workRemoteCache, String myAddress, String mySite, ExecutorService listenersExecutor) { protected InfinispanNotificationsManager(Cache<String, Serializable> workCache, RemoteCache<String, Serializable> workRemoteCache, String myAddress, String mySite, ExecutorService listenersExecutor) {
this.workCache = workCache; this.workCache = workCache;
this.workRemoteCache = workRemoteCache; this.workRemoteCache = workRemoteCache;
this.myAddress = myAddress; this.myAddress = myAddress;
@ -91,7 +90,7 @@ public class InfinispanNotificationsManager {
// Create and init manager including all listeners etc // Create and init manager including all listeners etc
public static InfinispanNotificationsManager create(KeycloakSession session, Cache<String, Serializable> workCache, String myAddress, String mySite, Set<RemoteStore> remoteStores) { public static InfinispanNotificationsManager create(KeycloakSession session, Cache<String, Serializable> workCache, String myAddress, String mySite, Set<RemoteStore> remoteStores) {
RemoteCache workRemoteCache = null; RemoteCache<String, Serializable> workRemoteCache = null;
if (!remoteStores.isEmpty()) { if (!remoteStores.isEmpty()) {
RemoteStore remoteStore = remoteStores.iterator().next(); RemoteStore remoteStore = remoteStores.iterator().next();
@ -158,21 +157,11 @@ public class InfinispanNotificationsManager {
// Add directly to remoteCache. Will notify remote listeners on all nodes in all DCs // Add directly to remoteCache. Will notify remote listeners on all nodes in all DCs
Retry.executeWithBackoff((int iteration) -> { Retry.executeWithBackoff((int iteration) -> {
try { try {
/* workRemoteCache.put(eventKey, wrappedEvent, 120, TimeUnit.SECONDS);
workaround for Infinispan 12.1.7.Final to prevent a deadlock while
DefaultInfinispanConnectionProviderFactory is shutting down PersistenceManagerImpl
that acquires a writeLock and this put that acquires a readLock.
First seen with https://issues.redhat.com/browse/ISPN-13664 and still occurs probably due to
https://issues.redhat.com/browse/ISPN-13666 in 13.0.10
Tracked in https://github.com/keycloak/keycloak/issues/9871
*/
synchronized (DefaultInfinispanConnectionProviderFactory.class) {
workRemoteCache.put(eventKey, wrappedEvent, 120, TimeUnit.SECONDS);
}
} catch (HotRodClientException re) { } catch (HotRodClientException re) {
if (logger.isDebugEnabled()) { if (logger.isDebugEnabled()) {
logger.debugf(re, "Failed sending notification to remote cache '%s'. Key: '%s', iteration '%s'. Will try to retry the task", logger.debugf(re, "Failed sending notification to remote cache '%s'. Key: '%s', iteration '%s'. Will try to retry the task",
workRemoteCache.getName(), eventKey, iteration); workRemoteCache.getName(), eventKey, iteration);
} }
// Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation. // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation.
@ -195,7 +184,7 @@ public class InfinispanNotificationsManager {
@CacheEntryModified @CacheEntryModified
public void cacheEntryModified(CacheEntryModifiedEvent<String, Serializable> event) { public void cacheEntryModified(CacheEntryModifiedEvent<String, Serializable> event) {
eventReceived(event.getKey(), event.getValue()); eventReceived(event.getKey(), event.getNewValue());
} }
@CacheEntryRemoved @CacheEntryRemoved
@ -209,30 +198,30 @@ public class InfinispanNotificationsManager {
@ClientListener @ClientListener
public class HotRodListener { public class HotRodListener {
private final RemoteCache<Object, Object> remoteCache; private final RemoteCache<String, Serializable> remoteCache;
public HotRodListener(RemoteCache<Object, Object> remoteCache) { public HotRodListener(RemoteCache<String, Serializable> remoteCache) {
this.remoteCache = remoteCache; this.remoteCache = remoteCache;
} }
@ClientCacheEntryCreated @ClientCacheEntryCreated
public void created(ClientCacheEntryCreatedEvent event) { public void created(ClientCacheEntryCreatedEvent<String> event) {
String key = event.getKey().toString(); String key = event.getKey();
hotrodEventReceived(key); hotrodEventReceived(key);
} }
@ClientCacheEntryModified @ClientCacheEntryModified
public void updated(ClientCacheEntryModifiedEvent event) { public void updated(ClientCacheEntryModifiedEvent<String> event) {
String key = event.getKey().toString(); String key = event.getKey();
hotrodEventReceived(key); hotrodEventReceived(key);
} }
@ClientCacheEntryRemoved @ClientCacheEntryRemoved
public void removed(ClientCacheEntryRemovedEvent event) { public void removed(ClientCacheEntryRemovedEvent<String> event) {
String key = event.getKey().toString(); String key = event.getKey();
taskFinished(key, true); taskFinished(key, true);
} }
@ -241,21 +230,7 @@ public class InfinispanNotificationsManager {
// TODO: Look at CacheEventConverter stuff to possibly include value in the event and avoid additional remoteCache request // TODO: Look at CacheEventConverter stuff to possibly include value in the event and avoid additional remoteCache request
try { try {
listenersExecutor.submit(() -> { listenersExecutor.submit(() -> {
eventReceived(key, remoteCache.get(key));
/*
workaround for Infinispan 12.1.7.Final to prevent a deadlock while
DefaultInfinispanConnectionProviderFactory is shutting down PersistenceManagerImpl
that acquires a writeLock and this get that acquires a readLock.
First seen with https://issues.redhat.com/browse/ISPN-13664 and still occurs probably due to
https://issues.redhat.com/browse/ISPN-13666 in 13.0.10
Tracked in https://github.com/keycloak/keycloak/issues/9871
*/
Object value;
synchronized (DefaultInfinispanConnectionProviderFactory.class) {
value = remoteCache.get(key);
}
eventReceived(key, (Serializable) value);
}); });
} catch (RejectedExecutionException ree) { } catch (RejectedExecutionException ree) {
// server is shutting down or pool was terminated - don't throw errors // server is shutting down or pool was terminated - don't throw errors

22
model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java
View file

@ -87,23 +87,13 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
@Override @Override
public void close() { public void close() {
/* if (cacheManager != null && !containerManaged) {
workaround for Infinispan 12.1.7.Final to prevent a deadlock while cacheManager.stop();
DefaultInfinispanConnectionProviderFactory is shutting down PersistenceManagerImpl
that acquires a writeLock and this removal that acquires a readLock.
First seen with https://issues.redhat.com/browse/ISPN-13664 and still occurs probably due to
https://issues.redhat.com/browse/ISPN-13666 in 13.0.10
Tracked in https://github.com/keycloak/keycloak/issues/9871
*/
synchronized (DefaultInfinispanConnectionProviderFactory.class) {
if (cacheManager != null && !containerManaged) {
cacheManager.stop();
}
if (remoteCacheProvider != null) {
remoteCacheProvider.stop();
}
cacheManager = null;
} }
if (remoteCacheProvider != null) {
remoteCacheProvider.stop();
}
cacheManager = null;
} }
@Override @Override