KEYCLOAK-17377 Fix invalidation cluster tests (do not hide failures)

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -21,10 +21,7 @@ import java.util.Iterator;
 import java.util.ServiceLoader;
 import java.util.concurrent.TimeUnit;
 
-import org.infinispan.Cache;
 import org.infinispan.client.hotrod.ProtocolVersion;
-import org.infinispan.client.hotrod.RemoteCache;
-import org.infinispan.commons.configuration.Builder;
 import org.infinispan.commons.util.FileLookup;
 import org.infinispan.commons.util.FileLookupFactory;
 import org.infinispan.configuration.cache.CacheMode;
@@ -465,8 +462,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
                     System.setProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR, jgroupsUdpMcastAddr);
                 }
                 try {
-                    // Compatibility with Wildfly
+                    JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-keycloak-jgroups-udp.xml", this.getClass().getClassLoader()).openStream());
-                    JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()).openStream());
                     channel.setName(nodeName);
                     JGroupsTransport transport = new JGroupsTransport(channel);
 
@@ -482,7 +478,7 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
 
                     transportBuilder.jmx()
-                        .jmxDomain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName)
+                        .domain(InfinispanConnectionProvider.JMX_DOMAIN + "-" + nodeName)
                         .enable();
 
                     logger.infof("Configured jgroups transport with the channel name: %s", nodeName);

model/infinispan/src/main/resources/default-configs/default-keycloak-jgroups-udp.xml

@@ -0,0 +1,61 @@
+<config xmlns="urn:org:jgroups"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/jgroups-4.2.xsd">
+   <!-- This file has been adpoted from https://github.com/infinispan/infinispan/blob/master/core/src/main/resources/default-configs/default-jgroups-udp.xml -->
+   <!-- jgroups.udp.address is deprecated and will be removed, see ISPN-11867 -->
+   <UDP bind_addr="${jgroups.bind.address,jgroups.udp.address:127.0.0.1}"
+        bind_port="${jgroups.bind.port,jgroups.udp.port:0}"
+        mcast_addr="${jgroups.udp.mcast_addr,jgroups.mcast_addr:228.6.7.8}"
+        mcast_port="${jgroups.udp.mcast_port,jgroups.mcast_port:46655}"
+        tos="0"
+        ucast_send_buf_size="1m"
+        mcast_send_buf_size="1m"
+        ucast_recv_buf_size="20m"
+        mcast_recv_buf_size="25m"
+        ip_ttl="${jgroups.ip_ttl:2}"
+        thread_naming_pattern="pl"
+        enable_diagnostics="false"
+        bundler_type="no-bundler"
+        max_bundle_size="8500"
+
+        thread_pool.min_threads="${jgroups.thread_pool.min_threads:0}"
+        thread_pool.max_threads="${jgroups.thread_pool.max_threads:200}"
+        thread_pool.keep_alive_time="60000"
+   />
+   <PING num_discovery_runs="3"/>
+   <MERGE3 min_interval="10000"
+           max_interval="30000"
+   />
+   <FD_SOCK/>
+   <!-- Suspect node `timeout` to `timeout + timeout_check_interval` millis after the last heartbeat -->
+   <FD_ALL timeout="10000"
+           interval="2000"
+           timeout_check_interval="1000"
+   />
+   <VERIFY_SUSPECT timeout="1000"/>
+   <pbcast.NAKACK2 xmit_interval="100"
+                   xmit_table_num_rows="50"
+                   xmit_table_msgs_per_row="1024"
+                   xmit_table_max_compaction_time="30000"
+                   resend_last_seqno="true"
+   />
+   <UNICAST3 xmit_interval="100"
+             xmit_table_num_rows="50"
+             xmit_table_msgs_per_row="1024"
+             xmit_table_max_compaction_time="30000"
+   />
+   <pbcast.STABLE stability_delay="500"
+                  desired_avg_gossip="5000"
+                  max_bytes="1M"
+   />
+   <pbcast.GMS print_local_addr="false"
+               join_timeout="${jgroups.join_timeout:500}"
+   />
+   <UFC max_credits="4m"
+        min_threshold="0.40"
+   />
+   <MFC max_credits="4m"
+        min_threshold="0.40"
+   />
+   <FRAG3 frag_size="8000"/>
+</config>

services/src/main/java/org/keycloak/services/filters/AbstractRequestFilter.java

@@ -30,7 +30,7 @@ import org.keycloak.services.resources.KeycloakApplication;
 public abstract class AbstractRequestFilter {
 
     protected void filter(ClientConnection clientConnection, Consumer<KeycloakSession> next) {
-        KeycloakSessionFactory sessionFactory = KeycloakApplication.getSessionFactory();
+        KeycloakSessionFactory sessionFactory = getSessionFactory();
         KeycloakSession session = sessionFactory.create();
 
         KeycloakTransactionManager tx = session.getTransactionManager();
@@ -51,6 +51,10 @@ public abstract class AbstractRequestFilter {
         }
     }
 
+    protected KeycloakSessionFactory getSessionFactory() {
+        return KeycloakApplication.getSessionFactory();
+    }
+
     protected void close(KeycloakSession session) {
         KeycloakTransactionManager tx = session.getTransactionManager();
         if (tx.isActive()) {

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertow.java

@@ -39,7 +39,6 @@ import org.jboss.logging.Logger;
 import org.jboss.resteasy.plugins.server.servlet.ResteasyContextParameters;
 import org.jboss.resteasy.plugins.server.undertow.UndertowJaxrsServer;
 import org.jboss.resteasy.spi.ResteasyDeployment;
-import org.jboss.resteasy.spi.ResteasyProviderFactory;
 import org.jboss.shrinkwrap.api.Archive;
 import org.jboss.shrinkwrap.api.spec.WebArchive;
 import org.jboss.shrinkwrap.descriptor.api.Descriptor;
@@ -57,6 +56,7 @@ import org.keycloak.testsuite.utils.undertow.UndertowDeployerHelper;
 import org.keycloak.testsuite.utils.undertow.UndertowWarClassLoader;
 import org.keycloak.util.JsonSerialization;
 
+import io.undertow.servlet.api.InstanceHandle;
 import javax.servlet.DispatcherType;
 import javax.servlet.ServletException;
 import java.io.IOException;
@@ -64,6 +64,7 @@ import java.lang.reflect.Field;
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import javax.servlet.Filter;
 import org.xnio.Options;
 import org.xnio.SslClientAuthMode;
 
@@ -103,7 +104,20 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
         di.setDefaultServletConfig(new DefaultServletConfig(true));
         di.addWelcomePage("theme/keycloak/welcome/resources/index.html");
 
-        FilterInfo filter = Servlets.filter("SessionFilter", UndertowRequestFilter.class);
+        // This is needed as in case of clustered undertow, several undertow instances share the same JVM, hence the default
+        // way accessing the factory in the UndertowRequestFilter via static reference to KeycloakApplication does not work:
+        // There are several KeycloakApplication instances in the JVM with no classloader separation as in a full-blown server.
+        InstanceHandle<Filter> filterInstance = new InstanceHandle<Filter>() {
+            @Override
+            public Filter getInstance() {
+                return new UndertowRequestFilter(sessionFactory);
+            }
+
+            @Override
+            public void release() {
+            }
+        };
+        FilterInfo filter = Servlets.filter("SessionFilter", UndertowRequestFilter.class, () -> filterInstance);
         di.addFilter(filter);
         di.addFilterUrlMapping("SessionFilter", "/*", DispatcherType.REQUEST);
         filter.setAsyncSupported(true);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractInvalidationClusterTest.java

@@ -50,27 +50,36 @@ public abstract class AbstractInvalidationClusterTest<T, TR> extends AbstractClu
         T testEntity = createTestEntityRepresentation();
 
         // CREATE
+        log.info("(1) createEntityOnCurrentFailNode");
         testEntity = createEntityOnCurrentFailNode(testEntity);
 
         if (backendFailover) {
+            log.info("(2) failure");
             failure();
         }
 
+        log.info("(3) assertEntityOnSurvivorNodesEqualsTo");
         assertEntityOnSurvivorNodesEqualsTo(testEntity);
 
+        log.info("(4) failback");
         failback();
+        log.info("(5) iterateCurrentFailNode");
         iterateCurrentFailNode();
 
         // UPDATE(s)
+        log.info("(6) testEntityUpdates");
         testEntity = testEntityUpdates(testEntity, backendFailover);
 
         // DELETE 
+        log.info("(7) deleteEntityOnCurrentFailNode");
         deleteEntityOnCurrentFailNode(testEntity);
 
         if (backendFailover) {
+            log.info("(8) failure");
             failure();
         }
 
+        log.info("(9) assertEntityOnSurvivorNodesIsDeleted");
         assertEntityOnSurvivorNodesIsDeleted(testEntity);
     }
 
@@ -136,6 +145,7 @@ public abstract class AbstractInvalidationClusterTest<T, TR> extends AbstractClu
     protected void assertEntityOnSurvivorNodesEqualsTo(T testEntityOnFailNode) {
         boolean entityDiffers = false;
         for (ContainerInfo survivorNode : getCurrentSurvivorNodes()) {
+            log.debug(String.format("Attempt to verify %s on survivor %s (%s)", getEntityType(testEntityOnFailNode), survivorNode, survivorNode.getContextRoot()));
             T testEntityOnSurvivorNode = readEntity(testEntityOnFailNode, survivorNode);
 
             if (EqualsBuilder.reflectionEquals(sortFields(testEntityOnSurvivorNode), sortFields(testEntityOnFailNode), excludedComparisonFields)) {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/ClientInvalidationClusterTest.java

@@ -4,7 +4,6 @@ import org.apache.commons.lang.RandomStringUtils;
 import org.junit.Before;
 import org.keycloak.admin.client.resource.ClientResource;
 import org.keycloak.admin.client.resource.ClientsResource;
-import org.keycloak.common.util.Retry;
 import org.keycloak.representations.idm.ClientRepresentation;
 import org.keycloak.testsuite.admin.ApiUtil;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
@@ -59,38 +58,24 @@ public class ClientInvalidationClusterTest extends AbstractInvalidationClusterTe
 
     @Override
     protected ClientRepresentation readEntity(ClientRepresentation client, ContainerInfo node) {
-        ClientRepresentation u = Retry.call(new Retry.Supplier<ClientRepresentation>() {
+        ClientRepresentation u = null;
-            @Override
+        try {
-            public ClientRepresentation get(int iteration) {
+            u = entityResource(client, node).toRepresentation();
-                try {
+        } catch (NotFoundException nfe) {
-                    return entityResource(client, node).toRepresentation();
+            // expected when client doesn't exist
-                } catch (NotFoundException nfe) {
+        }
-                    return null;
-                }
-            }
-        }, 3, 5000);
         return u;
     }
 
     @Override
     protected ClientRepresentation updateEntity(ClientRepresentation client, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(client, node).update(client);
-            @Override
-            public void run() {
-                entityResource(client, node).update(client);
-            }
-        }, 3, 5000);
         return readEntity(client, node);
     }
 
     @Override
     protected void deleteEntity(ClientRepresentation client, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(client, node).remove();
-            @Override
-            public void run() {
-                entityResource(client, node).remove();
-            }
-        }, 3, 5000);
         assertNull(readEntity(client, node));
     }
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/GroupInvalidationClusterTest.java

@@ -4,7 +4,6 @@ import org.apache.commons.lang.RandomStringUtils;
 import org.junit.Before;
 import org.keycloak.admin.client.resource.GroupResource;
 import org.keycloak.admin.client.resource.GroupsResource;
-import org.keycloak.common.util.Retry;
 import org.keycloak.representations.idm.GroupRepresentation;
 import org.keycloak.testsuite.admin.ApiUtil;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
@@ -67,38 +66,24 @@ public class GroupInvalidationClusterTest extends AbstractInvalidationClusterTes
 
     @Override
     protected GroupRepresentation readEntity(GroupRepresentation group, ContainerInfo node) {
-        GroupRepresentation u = Retry.call(new Retry.Supplier<GroupRepresentation>() {
+        GroupRepresentation u = null;
-            @Override
+        try {
-            public GroupRepresentation get(int iteration) {
+            u = entityResource(group, node).toRepresentation();
-                try {
+        } catch (NotFoundException nfe) {
-                    return entityResource(group, node).toRepresentation();
+            // expected when group doesn't exist
-                } catch (NotFoundException nfe) {
+        }
-                    return null;
-                }
-            }
-        }, 3, 5000);
         return u;
     }
 
     @Override
     protected GroupRepresentation updateEntity(GroupRepresentation group, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(group, node).update(group);
-            @Override
-            public void run() {
-                entityResource(group, node).update(group);
-            }
-        }, 3, 5000);
         return readEntity(group, node);
     }
 
     @Override
     protected void deleteEntity(GroupRepresentation group, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(group, node).remove();
-            @Override
-            public void run() {
-                entityResource(group, node).remove();
-            }
-        }, 3, 5000);
         assertNull(readEntity(group, node));
     }
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RealmInvalidationClusterTest.java

@@ -2,7 +2,6 @@ package org.keycloak.testsuite.cluster;
 
 import org.keycloak.admin.client.resource.RealmResource;
 import org.keycloak.admin.client.resource.RealmsResource;
-import org.keycloak.common.util.Retry;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
 
@@ -44,16 +43,12 @@ public class RealmInvalidationClusterTest extends AbstractInvalidationClusterTes
 
     @Override
     protected RealmRepresentation readEntity(RealmRepresentation realm, ContainerInfo node) {
-        RealmRepresentation realmOnNode = Retry.call(new Retry.Supplier<RealmRepresentation>() {
+        RealmRepresentation realmOnNode = null;
-            @Override
+        try {
-            public RealmRepresentation get(int iteration) {
+            realmOnNode = entityResource(realm, node).toRepresentation();
-                try {
+        } catch (NotFoundException nfe) {
-                    return entityResource(realm, node).toRepresentation();
+            // expected if realm not found
-                } catch (NotFoundException nfe) {
+        }
-                    return null;
-                }
-            }
-        }, 3, 5000);
         return realmOnNode;
     }
 
@@ -63,12 +58,7 @@ public class RealmInvalidationClusterTest extends AbstractInvalidationClusterTes
     }
 
     private RealmRepresentation updateEntity(String realmName, RealmRepresentation realm, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(realmName, node).update(realm);
-            @Override
-            public void run() {
-                entityResource(realmName, node).update(realm);
-            }
-        }, 3, 5000);
         return readEntity(realm, node);
     }
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/RoleInvalidationClusterTest.java

@@ -3,7 +3,6 @@ package org.keycloak.testsuite.cluster;
 import org.apache.commons.lang.RandomStringUtils;
 import org.keycloak.admin.client.resource.RoleResource;
 import org.keycloak.admin.client.resource.RolesResource;
-import org.keycloak.common.util.Retry;
 import org.keycloak.representations.idm.RoleRepresentation;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
 
@@ -49,16 +48,11 @@ public class RoleInvalidationClusterTest extends AbstractInvalidationClusterTest
     @Override
     protected RoleRepresentation readEntity(RoleRepresentation role, ContainerInfo node) {
         RoleRepresentation u = null;
-        u = Retry.call(new Retry.Supplier<RoleRepresentation>() {
+        try {
-            @Override
+            u = entityResource(role, node).toRepresentation();
-            public RoleRepresentation get(int iteration) {
+        } catch (NotFoundException nfe) {
-                try {
+            // expected when role doesn't exist
-                    return entityResource(role, node).toRepresentation();
+        }
-                } catch (NotFoundException nfe) {
-                    return null;
-                }
-            }
-        }, 3, 5000);
         return u;
     }
 
@@ -68,23 +62,13 @@ public class RoleInvalidationClusterTest extends AbstractInvalidationClusterTest
     }
 
     private RoleRepresentation updateEntity(String roleName, RoleRepresentation role, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(roleName, node).update(role);
-            @Override
-            public void run() {
-                entityResource(roleName, node).update(role);
-            }
-        }, 3, 5000);
         return readEntity(role, node);
     }
 
     @Override
     protected void deleteEntity(RoleRepresentation role, ContainerInfo node) {
-        Retry.execute(new Runnable() {
+        entityResource(role, node).remove();
-            @Override
-            public void run() {
-                entityResource(role, node).remove();
-            }
-        }, 3, 5000);
         assertNull(readEntity(role, node));
     }
 

testsuite/integration-arquillian/tests/pom.xml

@@ -641,6 +641,7 @@
                             <keycloak.connectionsInfinispan.remoteStorePort>${keycloak.connectionsInfinispan.remoteStorePort}</keycloak.connectionsInfinispan.remoteStorePort>
                             <keycloak.connectionsInfinispan.remoteStorePort.2>${keycloak.connectionsInfinispan.remoteStorePort.2}</keycloak.connectionsInfinispan.remoteStorePort.2>
                             <keycloak.connectionsInfinispan.remoteStoreServer>${keycloak.connectionsInfinispan.remoteStoreServer}</keycloak.connectionsInfinispan.remoteStoreServer>
+                            <keycloak.connectionsInfinispan.sessionsOwners>${keycloak.connectionsInfinispan.sessionsOwners}</keycloak.connectionsInfinispan.sessionsOwners>
                             <keycloak.testsuite.logging.pattern>${keycloak.testsuite.logging.pattern}</keycloak.testsuite.logging.pattern>
 
                             <keycloak.connectionsJpa.url.crossdc>${keycloak.connectionsJpa.url.crossdc}</keycloak.connectionsJpa.url.crossdc>
@@ -771,6 +772,7 @@
                 <auth.server.quarkus.skip.unpack>false</auth.server.quarkus.skip.unpack>
                 <auth.server.undertow.skip.unpack>true</auth.server.undertow.skip.unpack>
                 <auth.server.jboss.skip.unpack>true</auth.server.jboss.skip.unpack>
+                <keycloak.connectionsInfinispan.sessionsOwners>2</keycloak.connectionsInfinispan.sessionsOwners>
             </properties>
             <build>
                 <plugins>
@@ -1246,6 +1248,7 @@
 
                 <auth.server.jboss.skip.unpack>true</auth.server.jboss.skip.unpack>
 
+                <keycloak.connectionsInfinispan.sessionsOwners>2</keycloak.connectionsInfinispan.sessionsOwners>
             </properties>
             <build>
                 <plugins>

testsuite/utils/src/main/java/org/keycloak/testsuite/KeycloakServer.java

@@ -37,6 +37,7 @@ import org.keycloak.services.managers.RealmManager;
 import org.keycloak.services.resources.KeycloakApplication;
 import org.keycloak.testsuite.util.cli.TestsuiteCLI;
 import org.keycloak.util.JsonSerialization;
+import io.undertow.servlet.api.InstanceHandle;
 import org.xnio.Options;
 import org.xnio.SslClientAuthMode;
 
@@ -58,6 +59,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import javax.servlet.Filter;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
@@ -406,7 +408,17 @@ public class KeycloakServer {
             // KEYCLOAK-14178
             deployment.setProperty(ResteasyContextParameters.RESTEASY_DISABLE_HTML_SANITIZER, true);
 
-            FilterInfo filter = Servlets.filter("SessionFilter", UndertowRequestFilter.class);
+            InstanceHandle<Filter> filterInstance = new InstanceHandle<Filter>() {
+                @Override
+                public Filter getInstance() {
+                    return new UndertowRequestFilter(sessionFactory);
+                }
+
+                @Override
+                public void release() {
+                }
+            };
+            FilterInfo filter = Servlets.filter("SessionFilter", UndertowRequestFilter.class, () -> filterInstance);
             filter.setAsyncSupported(true);
 
             di.addFilter(filter);

testsuite/utils/src/main/java/org/keycloak/testsuite/UndertowRequestFilter.java

@@ -27,10 +27,17 @@ import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 
 import org.keycloak.common.ClientConnection;
+import org.keycloak.models.KeycloakSessionFactory;
 import org.keycloak.services.filters.AbstractRequestFilter;
 
 public class UndertowRequestFilter extends AbstractRequestFilter implements Filter {
 
+    private final KeycloakSessionFactory factory;
+
+    public UndertowRequestFilter(KeycloakSessionFactory factory) {
+        this.factory = factory;
+    }
+
     @Override
     public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
             throws UnsupportedEncodingException {
@@ -81,6 +88,11 @@ public class UndertowRequestFilter extends AbstractRequestFilter implements Filt
         };
     }
 
+    @Override
+    protected KeycloakSessionFactory getSessionFactory() {
+        return this.factory;
+    }
+
     @Override
     public void init(FilterConfig filterConfig) {
     }