KEYCLOAK-1678 Added initial cluster test for 2-node cluster.

testsuite/integration-arquillian/tests/base/pom.xml

@@ -35,7 +35,10 @@
         <exclude.console>-</exclude.console>
         <exclude.account>-</exclude.account>
         <exclude.client>-</exclude.client>
-        <exclude.migration>-</exclude.migration>
+        <!--exclude migration tests by default, enabled by 'migration' profile in tests/pom.xml-->
+        <exclude.migration>**/migration/**/*Test.java</exclude.migration>
+        <!--exclude cluster tests by default, enabled by 'auth-server-*-cluster' profiles in tests/pom.xml-->
+        <exclude.cluster>**/cluster/**/*Test.java</exclude.cluster>
     </properties>
     
     <dependencies>
@@ -108,29 +111,4 @@
 
     </build>
     
-    <profiles>    
-        <profile>
-            <id>no-account</id>
-            <properties>
-                <!-- Exclude all account management tests. -->
-                <exclude.account>**/account/**/*Test.java</exclude.account>
-            </properties>
-        </profile>
-        <profile>
-            <id>no-client</id>
-            <properties>
-                <!-- Exclude all client tests. -->
-                <exclude.client>**/client/**/*Test.java</exclude.client>
-            </properties>
-        </profile>
-        <profile>
-            <id>adapters-only</id>
-            <properties>
-                <exclude.account>**/account/**/*Test.java</exclude.account>
-                <exclude.client>**/client/**/*Test.java</exclude.client>
-                <exclude.migration>**/migration/**/*Test.java</exclude.migration>
-            </properties>
-        </profile>
-    </profiles>
-    
 </project>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AppServerTestEnricher.java

@@ -40,9 +40,10 @@ public class AppServerTestEnricher {
         String appServerQ = (annotatedClass == null ? null
                 : annotatedClass.getAnnotation(AppServerContainer.class).value());
 
-        return appServerQ == null || appServerQ.isEmpty()
+        return annotatedClass == null ? null // no @AppServerContainer annotation --> no adapter test
-                ? getAuthServerQualifier() // app server == auth server
+                : (appServerQ == null || appServerQ.isEmpty() // @AppServerContainer annotation present but qualifier not set --> relative adapter test
-                : appServerQ;
+                        ? getAuthServerQualifier() // app server == auth server
+                        : appServerQ);
     }
 
     public static String getAppServerContextRoot() {
@@ -129,7 +130,7 @@ public class AppServerTestEnricher {
             String jbossHomePath = appServerInfo.getProperties().get("jbossHome");
 
             File bin = new File(jbossHomePath + "/bin");
-            
+
             File clientJar = new File(jbossHomePath + "/bin/client/jboss-cli-client.jar");
             if (!clientJar.exists()) {
                 clientJar = new File(jbossHomePath + "/bin/client/jboss-client.jar"); // AS7
@@ -137,7 +138,7 @@ public class AppServerTestEnricher {
             if (!clientJar.exists()) {
                 throw new IOException("JBoss CLI client JAR not found.");
             }
-            
+
             String command = "java -jar " + clientJar.getAbsolutePath();
             String adapterScript = "adapter-install.cli";
             String samlAdapterScript = "adapter-install-saml.cli";

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -110,7 +110,10 @@ public class AuthServerTestEnricher {
 
             boolean authServerCluster = authServerQualifier.endsWith("-cluster");
 
-            String authServerType = authServerQualifier.replaceAll("^auth-server-", "").replaceAll("-cluster$", "");
+            String authServerType = authServerQualifier.replaceAll("auth-server-", "").replaceAll("-cluster", "");
+
+            log.info("authServerType:" + authServerType);
+
             String authServerFrontend = authServerCluster
                     ? "auth-server-" + authServerType + "-balancer" // in cluster mode the load-balancer container serves as auth server frontend
                     : authServerQualifier; // single-node mode
@@ -133,7 +136,7 @@ public class AuthServerTestEnricher {
             if (suiteContext.getAuthServerInfo() == null) {
                 throw new RuntimeException(String.format("No auth server activated. A container matching '%s' needs to be enabled in arquillian.xml.", authServerFrontend));
             }
-            if (authServerCluster && !suiteContext.getAuthServerBackendsInfo().isEmpty()) {
+            if (authServerCluster && suiteContext.getAuthServerBackendsInfo().isEmpty()) {
                 throw new RuntimeException(String.format("No cluster backend nodes activated. Containers matching '%sN' need to be enabled in arquillian.xml.", authServerBackend));
             }
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/SuiteContext.java

@@ -93,9 +93,13 @@ public final class SuiteContext {
 
     @Override
     public String toString() {
+        String containers = "Auth server: " + (isAuthServerCluster() ? "\nFrontend: " : "")
+                + authServerInfo.getQualifier() + "\n";
+        for (ContainerInfo bInfo : getAuthServerBackendsInfo()) {
+            containers += "Backend: " + bInfo + "\n";
+        }
         return "SUITE CONTEXT:\n"
-                + "Auth server: " + authServerInfo.getQualifier() + "\n"
+                + containers;
-                +(isAuthServerCluster() ? "Auth server cluster: " + getAuthServerBackendsInfo().size() + " nodes+\n" : "");
     }
 
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/ContainersTest.java

@@ -1,34 +0,0 @@
-package org.keycloak.testsuite;
-
-import java.util.List;
-import org.jboss.arquillian.container.test.api.ContainerController;
-import org.jboss.arquillian.test.api.ArquillianResource;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-import org.keycloak.representations.idm.RealmRepresentation;
-import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
-
-/**
- *
- * @author tkyjovsk
- */
-public class ContainersTest extends AbstractKeycloakTest {
-
-    @ArquillianResource
-    ContainerController controller;
-
-    @Override
-    public void addTestRealms(List<RealmRepresentation> testRealms) {
-    }
-
-    
-    @Test
-    public void testAuthServer() {
-
-        log.info("AUTH SERVER should be started.");
-        assertTrue(controller.isStarted(AuthServerTestEnricher.getAuthServerQualifier()));
-        
-    }
-
-}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/AbstractClusterTest.java

@@ -0,0 +1,59 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import static org.junit.Assert.assertTrue;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.models.Constants;
+import org.keycloak.testsuite.AbstractKeycloakTest;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import static org.keycloak.testsuite.auth.page.AuthRealm.ADMIN;
+import static org.keycloak.testsuite.auth.page.AuthRealm.MASTER;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public abstract class AbstractClusterTest extends AbstractKeycloakTest {
+
+    @ArquillianResource
+    protected ContainerController controller;
+
+    protected List<Keycloak> backendAdminClients = new ArrayList<>();
+
+    public void startBackendNodes(int count) {
+        if (count < 0 || count > 10) {
+            throw new IllegalArgumentException();
+        }
+        assertTrue(suiteContext.getAuthServerBackendsInfo().size() >= count);
+        for (int i = 0; i < count; i++) {
+
+            ContainerInfo backendNode = suiteContext.getAuthServerBackendsInfo().get(i);
+
+            controller.start(backendNode.getQualifier());
+            assertTrue(controller.isStarted(backendNode.getQualifier()));
+
+            log.info("Initializing admin client for: '" + backendNode.getContextRoot() + "/auth'");
+            backendAdminClients.add(Keycloak.getInstance(backendNode.getContextRoot() + "/auth",
+                    MASTER, ADMIN, ADMIN, Constants.ADMIN_CLI_CLIENT_ID));
+        }
+    }
+
+    protected ContainerInfo backendInfo(int i) {
+        return suiteContext.getAuthServerBackendsInfo().get(i);
+    }
+
+    protected void startBackendNode(int i) {
+        String container = backendInfo(i).getQualifier();
+        if (!controller.isStarted(container)) {
+            controller.start(container);
+        }
+    }
+
+    protected void stopBackendNode(int i) {
+        controller.kill(backendInfo(i).getQualifier());
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/cluster/TwoNodeClusterTest.java

@@ -0,0 +1,131 @@
+package org.keycloak.testsuite.cluster;
+
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import org.junit.Before;
+import org.junit.Test;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import static org.keycloak.testsuite.auth.page.AuthRealm.TEST;
+import static org.keycloak.testsuite.util.WaitUtils.pause;
+
+/**
+ *
+ * @author tkyjovsk
+ */
+public class TwoNodeClusterTest extends AbstractClusterTest {
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+    }
+
+    @Before
+    public void beforeTwoNodeClusterTest() {
+        startBackendNodes(2);
+        pause(3000);
+    }
+
+    @Test
+    public void testRealm() {
+        testRealm(TEST, false);
+    }
+
+    @Test
+    public void testRealmWithFailover() {
+        testRealm(TEST + "_fo", true);
+    }
+
+    public void testRealm(String realm, boolean containerFailover) {
+        RealmRepresentation testRealm = new RealmRepresentation();
+        testRealm.setRealm(realm);
+        testRealm.setEnabled(true);
+
+        // CREATE on node1
+        log.info("Creating test realm via node1.");
+        backend1AdminClient().realms().create(testRealm);
+        log.info("Test realm created.");
+
+        // check if created on node1
+        RealmRepresentation testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
+        assertEquals(testRealmOnBackend1.getRealm(), testRealm.getRealm());
+        if (containerFailover) {
+            stopBackend1();
+        }
+
+        // check if created on node2
+        RealmRepresentation testRealmOnBackend2 = backend2AdminClient().realms().realm(realm).toRepresentation();
+        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
+        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
+
+        failback();
+
+        // UPDATE on node2
+        testRealmOnBackend2.setRealm(realm + "_updated");
+        backend2AdminClient().realms().realm(realm).update(testRealmOnBackend2);
+        if (containerFailover) {
+            stopBackend2();
+        }
+        // check if updated on node1
+        testRealmOnBackend1 = backend1AdminClient().realms().realm(realm).toRepresentation();
+        assertEquals(testRealmOnBackend1.getId(), testRealmOnBackend2.getId());
+        assertEquals(testRealmOnBackend1.getRealm(), testRealmOnBackend2.getRealm());
+
+        failback();
+
+        // DELETE on node1
+        backend1AdminClient().realms().realm(realm).remove();
+        if (containerFailover) {
+            stopBackend1();
+        }
+        // check if deleted on node2
+        boolean testRealmOnBackend2Exists = false;
+        for (RealmRepresentation realmOnBackend2 : backend2AdminClient().realms().findAll()) {
+            if (realm.equals(realmOnBackend2.getRealm())
+                    || testRealmOnBackend1.getId().equals(realmOnBackend2.getId())) {
+                testRealmOnBackend2Exists = true;
+                break;
+            }
+        }
+        assertFalse(testRealmOnBackend2Exists);
+    }
+
+    protected ContainerInfo backend1Info() {
+        return backendInfo(0);
+    }
+
+    protected ContainerInfo backend2Info() {
+        return backendInfo(1);
+    }
+
+    protected Keycloak backend1AdminClient() {
+        return backendAdminClients.get(0);
+    }
+
+    protected Keycloak backend2AdminClient() {
+        return backendAdminClients.get(1);
+    }
+
+    protected void startBackend1() {
+        startBackendNode(0);
+    }
+
+    protected void startBackend2() {
+        startBackendNode(1);
+    }
+
+    protected void failback() {
+        startBackend1();
+        startBackend2();
+    }
+
+    protected void stopBackend1() {
+        stopBackendNode(0);
+    }
+
+    protected void stopBackend2() {
+        stopBackendNode(1);
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -86,11 +86,14 @@
                 <property name="enabled">${auth.server.wildfly.cluster}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend1.home}</property>
+                <property name="serverConfig">standalone-ha.xml</property>
                 <property name="javaVmArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend1.port.offset} 
                     -Xms64m -Xmx512m -XX:MaxPermSize=256m 
                     ${adapter.test.props}
+                    -Djboss.node.name=node1
                 </property>
+                <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>
@@ -100,11 +103,14 @@
                 <property name="enabled">${auth.server.wildfly.cluster}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
                 <property name="jbossHome">${keycloak.backend2.home}</property>
+                <property name="serverConfig">standalone-ha.xml</property>
                 <property name="javaVmArguments">
                     -Djboss.socket.binding.port-offset=${auth.server.backend2.port.offset} 
                     -Xms64m -Xmx512m -XX:MaxPermSize=256m 
                     ${adapter.test.props}
+                    -Djboss.node.name=node2
                 </property>
+                <!--<property name="outputToConsole">false</property>-->
                 <property name="managementPort">${auth.server.backend2.management.port}</property>
                 <property name="startupTimeoutInSeconds">${startup.timeout.sec}</property>
             </configuration>

testsuite/integration-arquillian/tests/pom.xml

@@ -159,7 +159,27 @@
     </build>
 
     <profiles>
-        
+
+        <profile>
+            <id>no-account</id>
+            <properties>
+                <exclude.account>**/account/**/*Test.java</exclude.account>
+            </properties>
+        </profile>
+        <profile>
+            <id>no-client</id>
+            <properties>
+                <exclude.client>**/client/**/*Test.java</exclude.client>
+            </properties>
+        </profile>
+        <profile>
+            <id>no-base</id>
+            <properties>
+                <exclude.account>**/account/**/*Test.java</exclude.account>
+                <exclude.client>**/client/**/*Test.java</exclude.client>
+            </properties>
+        </profile>
+                
         <profile>
             <id>common-test-dependencies</id>
             <activation>
@@ -456,12 +476,15 @@
         <profile>
             <id>auth-server-wildfly-cluster</id>
             <properties>
+                <!--disable exclusion pattern for cluster test which is enabled by default in base/pom.xml-->
+                <exclude.cluster>-</exclude.cluster>
+                
                 <auth.server.container>auth-server-wildfly-cluster</auth.server.container>
                 <startup.timeout.sec>300</startup.timeout.sec>
                 <adapter.test.props/>
                 <h2.version>1.3.173</h2.version>
 
-                <keycloak.balancer.home>${containers.home}/balancer/wildfly-${project.version}</keycloak.balancer.home>
+                <keycloak.balancer.home>${containers.home}/balancer/wildfly-balancer-${project.version}</keycloak.balancer.home>
                 <keycloak.backend1.home>${containers.home}/node1/keycloak-${project.version}</keycloak.backend1.home>
                 <keycloak.backend2.home>${containers.home}/node2/keycloak-${project.version}</keycloak.backend2.home>
 
@@ -482,6 +505,8 @@
                             <artifactId>maven-surefire-plugin</artifactId>
                             <configuration>
                                 <systemPropertyVariables>
+                                    <run.h2>true</run.h2>
+                                    
                                     <auth.server.wildfly.cluster>true</auth.server.wildfly.cluster>
                                     <auth.server.undertow>false</auth.server.undertow>
                                     <adapter.test.props>${adapter.test.props}</adapter.test.props>
@@ -495,7 +520,7 @@
                                     <auth.server.backend2.port.offset>102</auth.server.backend2.port.offset>
                                     <!--8180-->
                                     <auth.server.backend1.http.port>8181</auth.server.backend1.http.port>
-                                    <auth.server.backend2.http.port>8181</auth.server.backend2.http.port>
+                                    <auth.server.backend2.http.port>8182</auth.server.backend2.http.port>
                                     <!--8543-->
                                     <auth.server.backend1.https.port>8544</auth.server.backend1.https.port>
                                     <auth.server.backend2.https.port>8545</auth.server.backend2.https.port>
@@ -522,19 +547,24 @@
                                         <artifactItems>
                                             <artifactItem>
                                                 <groupId>org.keycloak.testsuite</groupId>
-                                                <artifactId>integration-arquillian-server-wildfly</artifactId>
+                                                <artifactId>integration-arquillian-server-wildfly-balancer</artifactId>
                                                 <version>${project.version}</version>
                                                 <type>zip</type>
-                                                <classifier>balancer</classifier>
+                                                <outputDirectory>${containers.home}/balancer</outputDirectory>
-                                                <outputDirectory>${containers.home}</outputDirectory>
                                             </artifactItem>
                                             <artifactItem>
                                                 <groupId>org.keycloak.testsuite</groupId>
                                                 <artifactId>integration-arquillian-server-wildfly</artifactId>
                                                 <version>${project.version}</version>
                                                 <type>zip</type>
-                                                <classifier>backend</classifier>
+                                                <outputDirectory>${containers.home}/node1</outputDirectory>
-                                                <outputDirectory>${containers.home}</outputDirectory>
+                                            </artifactItem>
+                                            <artifactItem>
+                                                <groupId>org.keycloak.testsuite</groupId>
+                                                <artifactId>integration-arquillian-server-wildfly</artifactId>
+                                                <version>${project.version}</version>
+                                                <type>zip</type>
+                                                <outputDirectory>${containers.home}/node2</outputDirectory>
                                             </artifactItem>
                                         </artifactItems>
                                         <overWriteIfNewer>true</overWriteIfNewer>
@@ -635,6 +665,10 @@
                     <name>migrated.auth.server.version</name>
                 </property>
             </activation>
+            <properties>
+                <!--diable exclusion pattern for migration tests, which is enabled by default in the base/pom.xml-->
+                <exclude.migration>-</exclude.migration>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -840,7 +874,7 @@
                 </pluginManagement>
             </build>
         </profile>
-        
+                
     </profiles>
     
 </project>