Merge pull request #2201 from tkyjovsk/cluster-testing

KEYCLOAK-1678 fixed containers setup & TwoNodeClusterTest

testsuite/integration-arquillian/README.md

@@ -13,17 +13,28 @@ other options are: `auth-server-wildfly` and `auth-server-eap7`. The values corr
 **Note 1:** For the non-default options it's necessary to build a corresponding server module prior to running any of the test modules.
 This can be done by building the server module directly (from `servers/wildfly`/`servers/eap7`), 
 or by activating `auth-server-wildfly`/`auth-server-eap7` profile when building from the top level module.
-(The profiles will also set the proper value of the `auth.server.container` property.)
 
 **Note 2:** Most server-side configurations are done during the build of the server module
 and included in the output artifact - which is then consumed by the test modules( if a corresponding profile is activated).
 To reflect a change in server config in the test (e.g. a datasource) it's necessary to rebuild the server module after each change.
 
-### Migration
+#### Migration
 
-Migration tests can be enabled by setting `-Dmigrated.auth.server.container` property or activating a corresponding profile.
+Migration tests can be enabled by setting `-Dmigrated.auth.server.version` property. Supported versions can be found at the bottom of `tests/pom.xml`.
-When enabled, the `AuthServerTestEnricher` class will start/stop the selected *migrated* instance 
+When enabled, the `AuthServerTestEnricher` class will start and stop the selected migrated instance 
-even **before** the *current* auth server instance is started.
+*before* the current auth server instance is started.
+
+#### Cluster Setup
+
+Cluster setup can be enabled with profile `auth-server-wildfly-cluster`.
+(It is also necessary to build the server modules with this profile before running the test. See *Notes 1 and 2* above.)
+
+Clustering tests require MULTICAST to be enabled on machine's `loopback` network interface.
+This can be done by running the following commands under root privileges:
+```
+route add -net 224.0.0.0 netmask 240.0.0.0 dev lo
+ifconfig lo multicast
+```
 
 ### App Servers
 

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java

@@ -3,12 +3,15 @@ package org.keycloak.testsuite.cluster;
 import java.util.ArrayList;
 import java.util.List;
 import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.graphene.page.Page;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import static org.junit.Assert.assertTrue;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
+import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.AbstractKeycloakTest;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.auth.page.AuthRealm;
 import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
 import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
 
@@ -35,25 +38,38 @@ public abstract class AbstractClusterTest extends AbstractKeycloakTest {
             controller.start(backendNode.getQualifier());
             assertTrue(controller.isStarted(backendNode.getQualifier()));
 
-            log.info("Initializing admin client for: '" + backendNode.getContextRoot() + "/auth'");
+            backendAdminClients.add(createAdminClientFor(backendNode));
-            backendAdminClients.add(Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
-                    MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID));
         }
     }
 
-    protected ContainerInfo backendInfo(int i) {
+    protected Keycloak createAdminClientFor(ContainerInfo backendNode) {
+        log.info("Initializing admin client for " + backendNode.getContextRoot() + "/auth");
+        return Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
+                MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+
+    protected ContainerInfo backendNode(int i) {
         return suiteContext.getAuthServerBackendsInfo().get(i);
     }
 
     protected void startBackendNode(int i) {
-        String container = backendInfo(i).getQualifier();
+        String container = backendNode(i).getQualifier();
         if (!controller.isStarted(container)) {
             controller.start(container);
+            backendAdminClients.set(i, createAdminClientFor(backendNode(i)));
         }
     }
 
-    protected void stopBackendNode(int i) {
+    protected void killBackendNode(int i) {
-        controller.kill(backendInfo(i).getQualifier());
+        backendAdminClients.get(i).close();
+        controller.kill(backendNode(i).getQualifier());
     }
 
+    protected void listRealms(int i) {
+        log.info(String.format("Node %s: AccessTokenString: %s", i + 1, backendAdminClients.get(i).tokenManager().getAccessTokenString()));
+        for (RealmRepresentation r : backendAdminClients.get(i).realms().findAll()) {
+            log.info(String.format("Node %s: Realm: %s, Id: %s", i + 1, r.getRealm(), r.getId()));
+        }
+    }
+    
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractTwoNodeClusterTest.java

@@ -0,0 +1,57 @@
+package org.keycloak.testsuite.cluster;
+
+import org.junit.Before;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractTwoNodeClusterTest extends AbstractClusterTest {
+
+    @Before
+    public void beforeTwoNodeClusterTest() {
+        startBackendNodes(2);
+        pause(3000);
+    }
+
+    protected ContainerInfo backend1Info() {
+        return backendNode(0);
+    }
+
+    protected ContainerInfo backend2Info() {
+        return backendNode(1);
+    }
+
+    protected Keycloak backend1AdminClient() {
+        return backendAdminClients.get(0);
+    }
+
+    protected Keycloak backend2AdminClient() {
+        return backendAdminClients.get(1);
+    }
+
+    protected void startBackend1() {
+        startBackendNode(0);
+    }
+
+    protected void startBackend2() {
+        startBackendNode(1);
+    }
+
+    protected void failback() {
+        startBackend1();
+        startBackend2();
+    }
+
+    protected void killBackend1() {
+        killBackendNode(0);
+    }
+
+    protected void killBackend2() {
+        killBackendNode(1);
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/EntityInvalidationClusterTest.java

@@ -3,55 +3,43 @@ package org.keycloak.testsuite.cluster;
 import java.util.List;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import org.junit.Before;
 import org.junit.Test;
-import org.keycloak.admin.client.Keycloak;
 import org.keycloak.representations.idm.RealmRepresentation;
-import org.keycloak.testsuite.arquillian.ContainerInfo;
 import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
-import static org.keycloak.testsuite.util.WaitUtils.pause;
 
 /**
  *
  * @author tkyjovsk
  */
-public class TwoNodeClusterTest extends AbstractClusterTest {
+public class EntityInvalidationClusterTest extends AbstractTwoNodeClusterTest {
 
     @Override
     public void addTestRealms(List<RealmRepresentation> testRealms) {
     }
 
-    @Before
+    @Test
-    public void beforeTwoNodeClusterTest() {
+    public void realmCRUDWithoutFailover() {
-        startBackendNodes(2);
+        realmCRUD(TEST + "_wofo", false);
-        pause(3000);
     }
 
     @Test
-    public void testRealm() {
+    public void realmCRUDWithFailover() {
-        testRealm(TEST, false);
+        realmCRUD(TEST + "_wfo", true);
     }
 
-    @Test
+    public void realmCRUD(String realm, boolean containerFailover) {
-    public void testRealmWithFailover() {
-        testRealm(TEST + "_fo", true);
-    }
-
-    public void testRealm(String realm, boolean containerFailover) {
         RealmRepresentation testRealm = new RealmRepresentation();
         testRealm.setRealm(realm);
         testRealm.setEnabled(true);
 
         // CREATE on node1
-        log.info("Creating test realm via node1.");
         backend1AdminClient().realms().create(testRealm);
-        log.info("Test realm created.");
 
         // check if created on node1
         RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
         assertEquals(testRealmOnBackend1.getRealm(), testRealm.getRealm());
         if (containerFailover) {
-            stopBackend1();
+            killBackend1();
         }
 
         // check if created on node2
@@ -62,27 +50,28 @@ public class TwoNodeClusterTest extends AbstractClusterTest {
         failback();
 
         // UPDATE on node2
-        testRealmOnBackend2.setRealm(realm + "_updated");
+        String realmUpdated = realm + "_updated";
+        testRealmOnBackend2.setRealm(realmUpdated);
         backend2AdminClient().realms().realm(realm).update(testRealmOnBackend2);
         if (containerFailover) {
-            stopBackend2();
+            killBackend2();
         }
         // check if updated on node1
-        testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
+        testRealmOnBackend1 = backend1AdminClient().realms().realm(realmUpdated).toRepresentation();
         assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
         assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
 
         failback();
 
         // DELETE on node1
-        backend1AdminClient().realms().realm(realm).remove();
+        backend1AdminClient().realms().realm(realmUpdated).remove();
         if (containerFailover) {
-            stopBackend1();
+            killBackend1();
         }
         // check if deleted on node2
         boolean testRealmOnBackend2Exists = false;
         for (RealmRepresentation realmOnBackend2 : backend2AdminClient().realms().findAll()) {
-            if (realm.equals(realmOnBackend2.getRealm())
+            if (realmUpdated.equals(realmOnBackend2.getRealm())
                     || testRealmOnBackend1.getId().equals(realmOnBackend2.getId())) {
                 testRealmOnBackend2Exists = true;
                 break;
@@ -91,41 +80,30 @@ public class TwoNodeClusterTest extends AbstractClusterTest {
         assertFalse(testRealmOnBackend2Exists);
     }
 
-    protected ContainerInfo backend1Info() {
+    @Test
-        return backendInfo(0);
+    public void createRealmViaFrontend() {
-    }
+        String realm = TEST + "_fe";
 
-    protected ContainerInfo backend2Info() {
+        RealmRepresentation testRealm = new RealmRepresentation();
-        return backendInfo(1);
+        testRealm.setRealm(realm);
-    }
+        testRealm.setEnabled(true);
 
-    protected Keycloak backend1AdminClient() {
+        // CREATE on frontend
-        return backendAdminClients.get(0);
+        adminClient.realms().create(testRealm);
-    }
 
-    protected Keycloak backend2AdminClient() {
+        // check if created on frontend
-        return backendAdminClients.get(1);
+        RealmRepresentation testRealmOnFrontend = adminClient.realms().realm(realm).toRepresentation();
-    }
+        assertEquals(testRealmOnFrontend.getRealm(), testRealm.getRealm());
 
-    protected void startBackend1() {
+        // check if created on node1
-        startBackendNode(0);
+        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
-    }
+        assertEquals(testRealmOnBackend1.getId(), testRealmOnFrontend.getId());
+        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnFrontend.getRealm());
 
-    protected void startBackend2() {
+        // check if created on node2
-        startBackendNode(1);
+        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
+        assertEquals(testRealmOnBackend2.getId(), testRealmOnFrontend.getId());
+        assertEquals(testRealmOnBackend2.getRealm(), testRealmOnFrontend.getRealm());
     }
-
+    
-    protected void failback() {
-        startBackend1();
-        startBackend2();
-    }
-
-    protected void stopBackend1() {
-        stopBackendNode(0);
-    }
-
-    protected void stopBackend2() {
-        stopBackendNode(1);
-    }
-
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/SessionFailoverClusterTest.java

@@ -0,0 +1,95 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.keycloak.representations.idm.RealmRepresentation;
+import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
+import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlDoesntStartWith;
+import static org.keycloak.testsuite.util.URLAssert.assertCurrentUrlStartsWith;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+import org.openqa.selenium.Cookie;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class SessionFailoverClusterTest extends AbstractTwoNodeClusterTest {
+
+    public static final String KEYCLOAK_SESSION_COOKIE = "KEYCLOAK_SESSION";
+    public static final String KEYCLOAK_IDENTITY_COOKIE = "KEYCLOAK_IDENTITY";
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+    }
+
+    @Test
+    @Ignore("work in progress") // only works with owners="2" at the moment
+    public void sessionFailover() {
+        
+        // LOGOUT
+        accountPage.navigateTo();
+        driver.navigate().refresh();
+        pause(3000);
+        loginPage.form().login(ADMIN, ADMIN);
+        assertCurrentUrlStartsWith(accountPage);
+        
+        Cookie sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookie);
+
+        killBackend1();
+
+        // check if session survived backend failure
+        
+        driver.navigate().refresh();
+        pause(3000);
+        
+        assertCurrentUrlStartsWith(accountPage);
+        Cookie sessionCookieAfterFailover = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookieAfterFailover);
+        assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
+
+        failback();
+
+        // check if session survived backend failback
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlStartsWith(accountPage);
+        Cookie sessionCookieAfterFailback = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNotNull(sessionCookieAfterFailback);
+        assertEquals(sessionCookieAfterFailover.getValue(), sessionCookie.getValue());
+
+        // LOGOUT
+        accountPage.navigateTo();
+        accountPage.signOut();
+
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookie = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookie);
+
+        killBackend1();
+        
+        // check if session survived backend failure
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookieAfterFailover = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookieAfterFailover);
+        
+        failback();
+    
+        // check if session survived backend failback
+        driver.navigate().refresh();
+        pause(3000);
+        assertCurrentUrlDoesntStartWith(accountPage);
+        masterRealmPage.navigateTo();
+        sessionCookieAfterFailback = driver.manage().getCookieNamed(KEYCLOAK_SESSION_COOKIE);
+        assertNull(sessionCookieAfterFailback);
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -72,10 +72,12 @@
                 <property name="enabled">${auth.server.wildfly.cluster}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.balancer.home}</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
+                </property>
-                    ${adapter.test.props}
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <property name="managementPort">${auth.server.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
@@ -87,11 +89,14 @@
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend1.home}</property>
                 <property name="serverConfig">standalone-ha.xml</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend1.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
-                    ${adapter.test.props}
                     -Djboss.node.name=node1
+                    ${adapter.test.props}
+                </property>
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
@@ -104,11 +109,14 @@
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend2.home}</property>
                 <property name="serverConfig">standalone-ha.xml</property>
-                <property name="javaVmArguments">
+                <property name="jbossArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend2.port.offset} 
-                    -Xms64m -Xmx512m -XX:MaxPermSize=256m 
-                    ${adapter.test.props}
                     -Djboss.node.name=node2
+                    ${adapter.test.props}
+                </property>
+                <property name="javaVmArguments">
+                    -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m
+                    -Djava.net.preferIPv4Stack=true
                 </property>
                 <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend2.management.port}</property>