KEYCLOAK-7924 Speed-up crossdc tests

Co-Authored-By: Hynek Mlnarik <hmlnarik@redhat.com>

.travis.yml

@@ -15,8 +15,8 @@ env:
     - TESTS=server-group3
     - TESTS=server-group4
     - TESTS=old
-    - TESTS=crossdc1
+    - TESTS=crossdc-server
-    - TESTS=crossdc2
+    - TESTS=crossdc-adapter
 
 jdk:
   - oraclejdk8

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -249,7 +249,7 @@ public class AuthServerTestEnricher {
         }
 
         suiteContextProducer.set(suiteContext);
-        CacheServerTestEnricher.initializeSuiteContext(suiteContext);
+        CrossDCTestEnricher.initializeSuiteContext(suiteContext);
         log.info("\n\n" + suiteContext);
     }
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java

@@ -1,90 +0,0 @@
-/*
- * Copyright 2018 Red Hat, Inc. and/or its affiliates
- * and other contributors as indicated by the @author tags.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.keycloak.testsuite.arquillian;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.jboss.arquillian.container.test.api.ContainerController;
-import org.jboss.arquillian.core.api.Instance;
-import org.jboss.arquillian.core.spi.Validate;
-import org.jboss.arquillian.core.api.annotation.Inject;
-import org.jboss.arquillian.core.api.annotation.Observes;
-import org.jboss.arquillian.test.spi.event.suite.AfterClass;
-import org.jboss.logging.Logger;
-
-import org.keycloak.testsuite.crossdc.DC;
-
-/**
- *
- * @author vramik
- */
-public class CacheServerTestEnricher {
-
-    protected static final Logger log = Logger.getLogger(CacheServerTestEnricher.class);
-    private static SuiteContext suiteContext;
-
-    @Inject 
-    private Instance<ContainerController> containerController;
-
-    static void initializeSuiteContext(SuiteContext suiteContext) {
-        Validate.notNull(suiteContext, "Suite context cannot be null.");
-        CacheServerTestEnricher.suiteContext = suiteContext;
-    }
-
-    public void afterClass(@Observes(precedence = 4) AfterClass event) {
-        if (!suiteContext.getCacheServersInfo().isEmpty()) {
-            stopCacheServer(suiteContext.getCacheServersInfo().get(DC.FIRST.ordinal()));
-            stopCacheServer(suiteContext.getCacheServersInfo().get(DC.SECOND.ordinal()));
-        }
-    }
-
-    private void stopCacheServer(ContainerInfo cacheServer) {
-        if (containerController.get().isStarted(cacheServer.getQualifier())) {
-            log.infof("Stopping %s", cacheServer.getQualifier());
-
-            containerController.get().stop(cacheServer.getQualifier());
-
-            // Workaround for possible arquillian bug. Needs to cleanup dir manually
-            String setupCleanServerBaseDir = getContainerProperty(cacheServer, "setupCleanServerBaseDir");
-            String cleanServerBaseDir = getContainerProperty(cacheServer, "cleanServerBaseDir");
-
-            if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
-                log.infof("Going to clean directory: %s", cleanServerBaseDir);
-
-                File dir = new File(cleanServerBaseDir);
-                if (dir.exists()) {
-                    try {
-                        dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));
-
-                        File deploymentsDir = new File(dir, "deployments");
-                        FileUtils.forceMkdir(deploymentsDir);
-                    } catch (IOException ioe) {
-                        throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
-                    }
-                }
-            }
-
-            log.infof("Stopped %s", cacheServer.getQualifier());
-        }
-    }
-
-    private String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
-        return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
-    }
-}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CrossDCTestEnricher.java

@@ -0,0 +1,329 @@
+/*
+ * Copyright 2018 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import static org.hamcrest.Matchers.lessThan;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.spi.Validate;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.After;
+import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
+import org.jboss.arquillian.test.spi.event.suite.Before;
+import org.jboss.logging.Logger;
+import static org.junit.Assert.assertThat;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.models.Constants;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
+import org.keycloak.testsuite.auth.page.AuthRealm;
+import org.keycloak.testsuite.client.KeycloakTestingClient;
+import org.keycloak.testsuite.crossdc.DC;
+import org.keycloak.testsuite.crossdc.ServerSetup;
+import java.util.Collection;
+import java.util.function.Consumer;
+import org.jboss.arquillian.container.spi.event.StopSuiteContainers;
+
+/**
+ *
+ * @author vramik
+ */
+public class CrossDCTestEnricher {
+
+    protected static final Logger log = Logger.getLogger(CrossDCTestEnricher.class);
+    private static SuiteContext suiteContext;
+
+    @Inject
+    private static Instance<ContainerController> containerController;
+
+    private static final Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
+    private static final Map<ContainerInfo, KeycloakTestingClient> backendTestingClients = new HashMap<>();
+
+    static void initializeSuiteContext(SuiteContext suiteContext) {
+        Validate.notNull(suiteContext, "Suite context cannot be null.");
+        CrossDCTestEnricher.suiteContext = suiteContext;
+    }
+
+    public void beforeTest(@Observes(precedence = -2) Before event) {
+        if (!suiteContext.isAuthServerCrossDc()) return;
+
+        //if annotation is present on method
+        InitialDcState annotation = event.getTestMethod().getAnnotation(InitialDcState.class);
+        
+        //annotation not present on method, taking it from class
+        if (annotation == null) {
+            Class<?> annotatedClass = getNearestSuperclassWithAnnotation(event.getTestClass().getJavaClass(), InitialDcState.class);
+
+            annotation = annotatedClass.getAnnotation(InitialDcState.class);
+        }
+
+        if (annotation == null) {
+            log.debug("No environment preparation requested, not changing auth/cache server run status.");
+            return; // Test does not specify its environment, so it's on its own
+        }
+
+        ServerSetup cacheServers = annotation.cacheServers();
+        ServerSetup authServers = annotation.authServers();
+
+        switch (cacheServers) {
+            case ALL_NODES_IN_EVERY_DC:
+            case FIRST_NODE_IN_EVERY_DC: //the same as ALL_NODES_IN_EVERY_DC as there is only one cache server per DC
+            case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC:
+                DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer);
+                break;
+
+            case FIRST_NODE_IN_FIRST_DC:
+            case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC:
+                startCacheServer(DC.FIRST);
+                stopCacheServer(DC.SECOND);
+                break;
+        }
+
+        switch (authServers) {
+            case ALL_NODES_IN_EVERY_DC:
+                forAllBackendNodes(CrossDCTestEnricher::startAuthServerBackendNode);
+                break;
+            case FIRST_NODE_IN_EVERY_DC:
+                DC.validDcsStream().forEach((DC dc) -> startAuthServerBackendNode(dc, 0));
+                DC.validDcsStream().forEach((DC dc) -> stopAuthServerBackendNode(dc, 1));
+                break;
+
+            case FIRST_NODE_IN_FIRST_DC:
+                startAuthServerBackendNode(DC.FIRST, 0);
+                stopAuthServerBackendNode(DC.FIRST, 1);
+                forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode);
+                break;
+
+            case ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC:
+                forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode);
+                startAuthServerBackendNode(DC.SECOND, 0);
+                stopAuthServerBackendNode(DC.SECOND, 1);
+                break;
+
+            case ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC:
+                forAllBackendNodesInDc(DC.FIRST, CrossDCTestEnricher::startAuthServerBackendNode);
+                forAllBackendNodesInDc(DC.SECOND, CrossDCTestEnricher::stopAuthServerBackendNode);
+                break;
+        }
+        
+        suspendPeriodicTasks();
+    }
+    
+    public void afterTest(@Observes After event) {
+        if (!suiteContext.isAuthServerCrossDc()) return;
+
+        restorePeriodicTasks();
+    }
+
+    public void stopSuiteContainers(@Observes(precedence = 4) StopSuiteContainers event) {
+        if (!suiteContext.isAuthServerCrossDc()) return;
+
+        DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer);
+        forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode);
+    }
+
+    private static void createRESTClientsForNode(ContainerInfo node) {
+        if (!backendAdminClients.containsKey(node)) {
+            backendAdminClients.put(node, createAdminClientFor(node));
+        }
+
+        if (!backendTestingClients.containsKey(node)) {
+            backendTestingClients.put(node, createTestingClientFor(node));
+        }
+    }
+    
+    private static void removeRESTClientsForNode(ContainerInfo node) {
+        if (backendAdminClients.containsKey(node)) {
+            backendAdminClients.get(node).close();
+            backendAdminClients.remove(node);
+        }
+
+        if (backendTestingClients.containsKey(node)) {
+            backendTestingClients.get(node).close();
+            backendTestingClients.remove(node);
+        }
+    }
+
+    public static Map<ContainerInfo, Keycloak> getBackendAdminClients() {
+        return Collections.unmodifiableMap(backendAdminClients);
+    }
+
+    public static Map<ContainerInfo, KeycloakTestingClient> getBackendTestingClients() {
+        return Collections.unmodifiableMap(backendTestingClients);
+    }
+
+    private static Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+    
+    private static KeycloakTestingClient createTestingClientFor(ContainerInfo node) {
+        log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth");
+        return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth");
+    }
+
+    // Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable.
+    private static void suspendPeriodicTasks() {
+        log.debug("--DC: suspendPeriodicTasks");
+        backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
+            testingClient.testing().suspendPeriodicTasks();
+        });
+    }
+
+    private static void restorePeriodicTasks() {
+        log.debug("--DC: restorePeriodicTasks");
+        backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
+            testingClient.testing().restorePeriodicTasks();
+        });
+    }
+
+    /**
+     * Returns cache server corresponding to given DC
+     * @param dc
+     * @return
+     */
+    private static ContainerInfo getCacheServer(DC dc) {
+        assertValidDc(dc);
+        int dcIndex = dc.ordinal();
+        return suiteContext.getCacheServersInfo().get(dcIndex);
+    }
+
+    private static void assertValidDc(DC dc) throws IllegalStateException {
+        if (dc == DC.UNDEFINED) {
+            throw new IllegalStateException("Invalid DC used: " + DC.UNDEFINED);
+        }
+    }
+
+    public static void startCacheServer(DC dc) {
+        if (!containerController.get().isStarted(getCacheServer(dc).getQualifier())) {
+            log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier());
+            containerController.get().start(getCacheServer(dc).getQualifier());
+            log.infof("--DC: Started %s", getCacheServer(dc).getQualifier());
+        }
+    }
+
+    public static void stopCacheServer(DC dc) {
+        String qualifier = getCacheServer(dc).getQualifier();
+
+        if (containerController.get().isStarted(qualifier)) {
+            log.infof("--DC: Stopping %s", qualifier);
+
+            containerController.get().stop(qualifier);
+
+            // Workaround for possible arquillian bug. Needs to cleanup dir manually 
+            String setupCleanServerBaseDir = getContainerProperty(getCacheServer(dc), "setupCleanServerBaseDir");
+            String cleanServerBaseDir = getContainerProperty(getCacheServer(dc), "cleanServerBaseDir");
+
+            if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
+                log.debugf("Going to clean directory: %s", cleanServerBaseDir);
+
+                File dir = new File(cleanServerBaseDir);
+                if (dir.exists()) {
+                    try {
+                        dir.renameTo(new File(dir.getParentFile(), dir.getName() + "-backup-" + System.currentTimeMillis()));
+
+                        File deploymentsDir = new File(dir, "deployments");
+                        FileUtils.forceMkdir(deploymentsDir);
+                    } catch (IOException ioe) {
+                        throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
+                    }
+                }
+            }
+
+            log.infof("--DC: Stopped %s", qualifier);
+        }
+    }
+
+    public static void forAllBackendNodes(Consumer<ContainerInfo> functionOnContainerInfo) {
+        suiteContext.getDcAuthServerBackendsInfo().stream()
+          .flatMap(Collection::stream)
+          .forEach(functionOnContainerInfo);
+    }
+
+    public static void forAllBackendNodesInDc(DC dc, Consumer<ContainerInfo> functionOnContainerInfo) {
+        assertValidDc(dc);
+        suiteContext.getDcAuthServerBackendsInfo().get(dc.ordinal()).stream()
+          .forEach(functionOnContainerInfo);
+    }
+
+    public static void stopAuthServerBackendNode(ContainerInfo containerInfo) {
+        if (containerInfo.isStarted()) {
+            log.infof("--DC: Stopping backend auth-server node: %s", containerInfo.getQualifier());
+            removeRESTClientsForNode(containerInfo);
+            containerController.get().stop(containerInfo.getQualifier());
+        }
+    }
+
+    public static void startAuthServerBackendNode(ContainerInfo containerInfo) {
+        if (! containerInfo.isStarted()) {
+            log.infof("--DC: Starting backend auth-server node: %s", containerInfo.getQualifier());
+            containerController.get().start(containerInfo.getQualifier());
+            createRESTClientsForNode(containerInfo);
+        }
+    }
+
+    public static ContainerInfo getBackendNode(DC dc, int nodeIndex) {
+        assertValidDc(dc);
+        int dcIndex = dc.ordinal();
+        assertThat((Integer) dcIndex, lessThan(suiteContext.getDcAuthServerBackendsInfo().size()));
+        final List<ContainerInfo> dcNodes = suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
+        return dcNodes.get(nodeIndex);
+    }
+
+    /**
+     * Starts a manually-controlled backend auth-server node in cross-DC scenario.
+     * @param dc
+     * @param nodeIndex
+     * @return Started instance descriptor.
+     */
+    public static ContainerInfo startAuthServerBackendNode(DC dc, int nodeIndex) {
+        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
+        startAuthServerBackendNode(dcNode);
+        return dcNode;
+    }
+
+    /**
+     * Stops a manually-controlled backend auth-server node in cross-DC scenario.
+     * @param dc
+     * @param nodeIndex
+     * @return Stopped instance descriptor.
+     */
+    public static ContainerInfo stopAuthServerBackendNode(DC dc, int nodeIndex) {
+        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
+        stopAuthServerBackendNode(dcNode);
+        return dcNode;
+    }
+
+    private Class getNearestSuperclassWithAnnotation(Class<?> testClass, Class annotationClass) {
+        return (testClass.isAnnotationPresent(annotationClass)) ? testClass
+                : (testClass.getSuperclass().equals(Object.class) ? null // stop recursion
+                : getNearestSuperclassWithAnnotation(testClass.getSuperclass(), annotationClass)); // continue recursion
+    }
+
+    private static String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
+        return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -66,7 +66,7 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .observer(JmxConnectorRegistryCreator.class)
                 .observer(AuthServerTestEnricher.class)
                 .observer(AppServerTestEnricher.class)
-                .observer(CacheServerTestEnricher.class)
+                .observer(CrossDCTestEnricher.class)
                 .observer(H2TestEnricher.class);
         builder
                 .service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/InitialDcState.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 201 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.arquillian.annotation;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import org.keycloak.testsuite.crossdc.ServerSetup;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Specifies initial state of auth-server and cache-server nodes before start of each test
+ * in multinode setup like in cross-DC tests.
+ * When a test class is annotated, this annotation is applied to every test method in the class
+ * but can be overridden on method level.
+ * 
+ * @author vramik
+ * @author hmlnarik
+ */
+@Documented
+@Retention(RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface InitialDcState {
+    ServerSetup cacheServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC;
+    ServerSetup authServers() default ServerSetup.FIRST_NODE_IN_EVERY_DC;
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/DC.java

@@ -16,6 +16,9 @@
  */
 package org.keycloak.testsuite.crossdc;
 
+import java.util.Arrays;
+import java.util.stream.Stream;
+
 /**
  * Identifier of datacentre in the testsuite
  * @author hmlnarik
@@ -28,4 +31,10 @@ public enum DC {
     public int getDcIndex() {
         return ordinal();
     }
+
+    private static final DC[] VALID_DCS = new DC[] { FIRST, SECOND };
+
+    public static Stream<DC> validDcsStream() {
+        return Arrays.stream(VALID_DCS);
+    }
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/crossdc/ServerSetup.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+/**
+ * @author vramik
+ */
+public enum ServerSetup {
+    FIRST_NODE_IN_FIRST_DC,
+    FIRST_NODE_IN_EVERY_DC,
+    ALL_NODES_IN_EVERY_DC,
+    ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC,
+    ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC;
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java

@@ -16,38 +16,31 @@
  */
 package org.keycloak.testsuite.crossdc;
 
-import org.apache.commons.io.FileUtils;
 import org.keycloak.admin.client.Keycloak;
-import org.keycloak.models.Constants;
 import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
 import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
 import org.keycloak.testsuite.arquillian.LoadBalancerController;
 import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
-import org.keycloak.testsuite.auth.page.AuthRealm;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import org.junit.After;
 import org.junit.Before;
 import org.keycloak.testsuite.client.KeycloakTestingClient;
 
-import static org.hamcrest.Matchers.lessThan;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
 
 /**
  * Abstract cross-data-centre test that defines primitives for handling cross-DC setup.
  * @author hmlnarik
  */
+@InitialDcState
 public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
 
     // Keep the following constants in sync with arquillian
@@ -59,59 +52,34 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
     @LoadBalancer(value = QUALIFIER_NODE_BALANCER)
     protected LoadBalancerController loadBalancerCtrl;
 
-    @ArquillianResource
-    protected ContainerController containerController;
-
-    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
-
-    protected Map<ContainerInfo, KeycloakTestingClient> backendTestingClients = new HashMap<>();
-
     @Before
     @Override
     public void beforeAbstractKeycloakTest() throws Exception {
-        log.debug("--DC: Starting both cache servers and first node from each DC");
-        startCacheServer(DC.FIRST);
-        startCacheServer(DC.SECOND);
-
-        startBackendNode(DC.FIRST, 0);
-        startBackendNode(DC.SECOND, 0);
-
-        initRESTClientsForStartedNodes();
-        suspendPeriodicTasks();
-
         enableOnlyFirstNodeInFirstDc();
 
         super.beforeAbstractKeycloakTest();
     }
 
-    @Override
-    public void deleteCookies() {
-        //Overrides AbstractTestRealmKeycloakTest.deleteCookies 
-        //as it tries to delete cookies in 'test' realm @After test
-        //when backend containers are stopped already.
-    }
-    
     @After
     @Override
     public void afterAbstractKeycloakTest() {
         log.debug("--DC: after AbstractCrossDCTest");
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0);    // make sure first node is started
         enableOnlyFirstNodeInFirstDc();
 
         super.afterAbstractKeycloakTest();
 
-        restorePeriodicTasks();
         removeTestRealms();
-        terminateStartedServers();
         loadBalancerCtrl.disableAllBackendNodes();
     }
 
     private void enableOnlyFirstNodeInFirstDc() {
         log.debug("--DC: Enable only first node in first datacenter");
         this.loadBalancerCtrl.disableAllBackendNodes();
-        if (!getBackendNode(DC.FIRST, 0).isStarted()) {
+        if (!CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).isStarted()) {
             throw new IllegalStateException("--DC: Trying to enable not started node on load-balancer");
         }
-        loadBalancerCtrl.enableBackendNodeByName(getBackendNode(DC.FIRST, 0).getQualifier());
+        loadBalancerCtrl.enableBackendNodeByName(CrossDCTestEnricher.getBackendNode(DC.FIRST, 0).getQualifier());
     }
 
     private void removeTestRealms() {
@@ -122,52 +90,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         testContext.setTestRealmReps(new ArrayList<>());
     }
 
-    protected void terminateStartedServers() {
-        log.debug("--DC: Halting all nodes that are started");
-        this.suiteContext.getDcAuthServerBackendsInfo().stream()
-            .flatMap(List::stream)
-            .filter(ContainerInfo::isStarted)
-            .forEach((ContainerInfo containerInfo) -> {
-                containerController.stop(containerInfo.getQualifier());
-                removeRESTClientsForNode(containerInfo);
-            });
-    }
-
-    private void initRESTClientsForStartedNodes() {
-        log.debug("--DC: Init REST clients for started nodes");
-        this.suiteContext.getDcAuthServerBackendsInfo().stream()
-                .flatMap(List::stream)
-                .filter(ContainerInfo::isStarted)
-                .forEach(containerInfo -> {
-                    createRESTClientsForNode(containerInfo);
-                });
-    }
-
-    // Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable.
-    private void suspendPeriodicTasks() {
-        log.debug("--DC: suspendPeriodicTasks");
-        backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
-            testingClient.testing().suspendPeriodicTasks();
-        });
-    }
-
-    private void restorePeriodicTasks() {
-        log.debug("--DC: restorePeriodicTasks");
-        backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
-            testingClient.testing().restorePeriodicTasks();
-        });
-    }
-
-    protected Keycloak createAdminClientFor(ContainerInfo node) {
-        log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth");
-        return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
-    }
-
-    protected KeycloakTestingClient createTestingClientFor(ContainerInfo node) {
-        log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth");
-        return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth");
-    }
-
     protected Keycloak getAdminClientForStartedNodeInDc(int dcIndex) {
         ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream()
                 .filter(ContainerInfo::isStarted)
@@ -182,14 +104,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      * @return
      */
     protected Keycloak getAdminClientFor(ContainerInfo node) {
-        Keycloak client = backendAdminClients.get(node);
+        Keycloak client = CrossDCTestEnricher.getBackendAdminClients().get(node);
         if (client == null && node.equals(suiteContext.getAuthServerInfo())) {
             client = this.adminClient;
         }
         return client;
     }
 
-
     protected KeycloakTestingClient getTestingClientForStartedNodeInDc(int dcIndex) {
         ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream()
                 .filter(ContainerInfo::isStarted)
@@ -204,35 +125,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      * @return
      */
     protected KeycloakTestingClient getTestingClientFor(ContainerInfo node) {
-        KeycloakTestingClient client = backendTestingClients.get(node);
+        KeycloakTestingClient client = CrossDCTestEnricher.getBackendTestingClients().get(node);
         if (client == null && node.equals(suiteContext.getAuthServerInfo())) {
             client = this.testingClient;
         }
         return client;
     }
 
-    protected void createRESTClientsForNode(ContainerInfo node) {
-        if (!backendAdminClients.containsKey(node)) {
-            backendAdminClients.put(node, createAdminClientFor(node));
-        }
-
-        if (!backendTestingClients.containsKey(node)) {
-            backendTestingClients.put(node, createTestingClientFor(node));
-        }
-    }
-
-    protected void removeRESTClientsForNode(ContainerInfo node) {
-        if (backendAdminClients.containsKey(node)) {
-            backendAdminClients.get(node).close();
-            backendAdminClients.remove(node);
-        }
-
-        if (backendTestingClients.containsKey(node)) {
-            backendTestingClients.get(node).close();
-            backendTestingClients.remove(node);
-        }
-    }
-
     /**
      * Disables routing requests to the given data center in the load balancer.
      * @param dc
@@ -293,97 +192,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
     }
 
-    /**
-     * Starts a manually-controlled backend auth-server node in cross-DC scenario.
-     * @param dc
-     * @param nodeIndex
-     * @return Started instance descriptor.
-     */
-    protected ContainerInfo startBackendNode(DC dc, int nodeIndex) {
-        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
-
-        assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
-
-        if (!containerController.isStarted(dcNode.getQualifier())) {
-            log.infof("--DC: Starting backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
-            containerController.start(dcNode.getQualifier());
-
-            createRESTClientsForNode(dcNode);
-        }
-        return dcNode;
-    }
-
-    /**
-     * Stops a manually-controlled backend auth-server node in cross-DC scenario.
-     * @param dc
-     * @param nodeIndex
-     * @return Stopped instance descriptor.
-     */
-    protected ContainerInfo stopBackendNode(DC dc, int nodeIndex) {
-        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
-
-        removeRESTClientsForNode(dcNode);
-
-        assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
-        
-        log.infof("--DC: Stopping backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
-        containerController.stop(dcNode.getQualifier());
-        return dcNode;
-    }
-
-    private ContainerInfo getBackendNode(DC dc, int nodeIndex) {
-        int dcIndex = dc.ordinal();
-        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
-        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
-        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
-        return dcNodes.get(nodeIndex);
-    }
-
-    /**
-     * Returns cache server corresponding to given DC
-     * @param dc
-     * @return
-     */
-    protected ContainerInfo getCacheServer(DC dc) {
-        int dcIndex = dc.ordinal();
-        return this.suiteContext.getCacheServersInfo().get(dcIndex);
-    }
-
-    protected void startCacheServer(DC dc) {
-        if (!containerController.isStarted(getCacheServer(dc).getQualifier())) {
-            log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier());
-            containerController.start(getCacheServer(dc).getQualifier());
-        }
-    }
-
-    protected void stopCacheServer(ContainerInfo cacheServer) {
-        log.infof("--DC: Stopping %s", cacheServer.getQualifier());
-
-        containerController.stop(cacheServer.getQualifier());
-
-        // Workaround for possible arquillian bug. Needs to cleanup dir manually
-        String setupCleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("setupCleanServerBaseDir");
-        String cleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("cleanServerBaseDir");
-
-        if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
-            log.infof("--DC: Going to clean directory: %s", cleanServerBaseDir);
-
-            File dir = new File(cleanServerBaseDir);
-            if (dir.exists()) {
-                try {
-                    dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));
-
-                    File deploymentsDir = new File(dir, "deployments");
-                    FileUtils.forceMkdir(deploymentsDir);
-                } catch (IOException ioe) {
-                    throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
-                }
-            }
-        }
-
-        log.infof("--DC: Stopped %s", cacheServer.getQualifier());
-    }
-
     /**
      * Sets time offset on all the started containers.
      *
@@ -396,7 +204,7 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
     }
 
     private void setTimeOffsetOnAllStartedContainers(int offset) {
-        backendTestingClients.entrySet().stream()
+        CrossDCTestEnricher.getBackendTestingClients().entrySet().stream()
                 .filter(testingClientEntry -> testingClientEntry.getKey().isStarted())
                 .map(testingClientEntry -> testingClientEntry.getValue())
                 .forEach(testingClient -> testingClient.testing().setTimeOffset(Collections.singletonMap("offset", String.valueOf(offset))));

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -50,6 +50,8 @@ import org.hamcrest.Matchers;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertThat;
+import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
 
 /**
  *
@@ -80,6 +82,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
     }
 
     @Test
+    @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC)
     public void sendResetPasswordEmailSuccessWorksInCrossDc(
       @JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node0Statistics,
       @JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics,
@@ -87,7 +90,6 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
       @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
         log.debug("--DC: START sendResetPasswordEmailSuccessWorksInCrossDc");
         
-        startBackendNode(DC.FIRST, 1);
         cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
 
         Comparable originalNumberOfEntries = cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES);
@@ -155,6 +157,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
     }
 
     @Test
+    @InitialDcState(authServers = ServerSetup.FIRST_NODE_IN_FIRST_DC)
     public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
         log.debug("--DC: START sendResetPasswordEmailAfterNewNodeAdded");
         disableDcOnLoadBalancer(DC.SECOND);
@@ -188,7 +191,8 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
         assertEquals("Your account has been updated.", PageUtils.getPageTitle(driver));
 
         disableDcOnLoadBalancer(DC.FIRST);
-        startBackendNode(DC.SECOND, 1);
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 1);
+        CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.SECOND, 1);
 
         Retry.execute(() -> {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java

@@ -37,10 +37,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.LaxRedirectStrategy;
 import org.junit.Test;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
+@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC)
 public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
 
     @ArquillianResource
@@ -56,35 +58,11 @@ public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
 
     @Override
     public void beforeAbstractKeycloakTestRealmImport() {
-        log.debug("--DC: Starting cacheServers if not started already");
+        loadBalancerCtrl.enableAllBackendNodes();
-        suiteContext.getCacheServersInfo().stream()
-                .filter((containerInfo) -> !containerInfo.isStarted())
-                .map(ContainerInfo::getQualifier)
-                .forEach(containerController::start);
-        
-        log.debug("--DC: Initializing load balancer - enabling all started nodes across DCs");
-        this.loadBalancerCtrl.disableAllBackendNodes();
-
-        this.suiteContext.getDcAuthServerBackendsInfo().stream()
-                .flatMap(List::stream)
-                .filter((containerInfo) -> !containerInfo.getQualifier().contains("manual"))
-                .filter((containerInfo) -> !containerInfo.isStarted())
-                .map(ContainerInfo::getQualifier)
-                .forEach((nodeName) -> {
-                    containerController.start(nodeName);
-                    loadBalancerCtrl.enableBackendNodeByName(nodeName);
-                });
     }
 
     @Override
     public void postAfterAbstractKeycloak() {
-        log.debug("--DC: postAfterAbstractKeycloak");
-        suiteContext.getDcAuthServerBackendsInfo().stream()
-                .flatMap(List::stream)
-                .filter(ContainerInfo::isStarted)
-                .map(ContainerInfo::getQualifier)
-                .forEach(containerController::stop);
-
         loadBalancerCtrl.disableAllBackendNodes();
         
         //realms is already removed and this prevents another removal in AuthServerTestEnricher.afterClass

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionExpirationCrossDCTest.java

@@ -40,7 +40,9 @@ import org.keycloak.representations.idm.UserSessionRepresentation;
 import org.keycloak.testsuite.Assert;
 import org.keycloak.common.util.Retry;
 import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
 import org.keycloak.testsuite.arquillian.InfinispanStatistics;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
 import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanCacheStatistics;
 import org.keycloak.testsuite.arquillian.annotation.JmxInfinispanChannelStatistics;
 import org.keycloak.testsuite.util.ClientBuilder;
@@ -417,20 +419,18 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
 
 
     @Test
+    @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_FIRST_NODE_IN_SECOND_DC)
     public void testLogoutUserWithFailover(
             @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
             @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
             @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
 
-        // Start node2 on first DC
-        startBackendNode(DC.FIRST, 1);
-
         // Don't include remote stats. Size is smaller because of distributed cache
         List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
                 false, cacheDc1Statistics, cacheDc2Statistics, false);
 
         // Kill node2 now. Around 10 sessions (half of SESSIONS_COUNT) will be lost on Keycloak side. But not on infinispan side
-        stopBackendNode(DC.FIRST, 1);
+        CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1);
 
         channelStatisticsCrossDc.reset();
 
@@ -461,15 +461,12 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
 
 
     @Test
+    @InitialDcState(authServers = ServerSetup.ALL_NODES_IN_EVERY_DC)
     public void testLogoutWithAllStartedNodes(
             @JmxInfinispanCacheStatistics(dc=DC.FIRST, managementPortProperty = "cache.server.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc1Statistics,
             @JmxInfinispanCacheStatistics(dc=DC.SECOND, managementPortProperty = "cache.server.2.management.port", cacheName=InfinispanConnectionProvider.USER_SESSION_CACHE_NAME) InfinispanStatistics cacheDc2Statistics,
             @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
 
-        // Start node2 on every DC
-        startBackendNode(DC.FIRST, 1);
-        startBackendNode(DC.SECOND, 1);
-
         // Create sessions. Don't include remote stats. Size is smaller because of distributed cache
         List<OAuthClient.AccessTokenResponse> responses = createInitialSessions(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME,
                 false, cacheDc1Statistics, cacheDc2Statistics, false);
@@ -505,8 +502,8 @@ public class SessionExpirationCrossDCTest extends AbstractAdminCrossDCTest {
         }, 50, 50);
 
         // Stop both nodes
-        stopBackendNode(DC.FIRST, 1);
+        CrossDCTestEnricher.stopAuthServerBackendNode(DC.FIRST, 1);
-        stopBackendNode(DC.SECOND, 1);
+        CrossDCTestEnricher.stopAuthServerBackendNode(DC.SECOND, 1);
     }
 
     private void assertTestAppActiveSessionsCount(int expectedSessionsCount) {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java

@@ -23,10 +23,13 @@ import java.util.List;
 import org.junit.Before;
 import org.junit.Test;
 import org.keycloak.OAuth2Constants;
+import org.keycloak.common.util.Retry;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.Assert;
 import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.CrossDCTestEnricher;
+import org.keycloak.testsuite.arquillian.annotation.InitialDcState;
 import org.keycloak.testsuite.util.OAuthClient;
 
 /**
@@ -34,38 +37,28 @@ import org.keycloak.testsuite.util.OAuthClient;
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
+@InitialDcState(authServers = ServerSetup.ALL_NODES_IN_FIRST_DC_NO_NODES_IN_SECOND_DC)
 public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
 
     private static final int SESSIONS_COUNT = 10;
 
     @Before
     public void beforeSessionsPreloadCrossDCTest() throws Exception {
-        // Start DC1 and only All Keycloak nodes on DC2 are stopped
-        stopBackendNode(DC.SECOND, 0);
         disableDcOnLoadBalancer(DC.SECOND);
     }
 
-
     private void stopAllCacheServersAndAuthServers() {
         log.infof("Going to stop all auth servers");
 
-        stopBackendNode(DC.FIRST, 0);
+        CrossDCTestEnricher.forAllBackendNodes(CrossDCTestEnricher::stopAuthServerBackendNode);
-        disableLoadBalancerNode(DC.FIRST, 0);
+        loadBalancerCtrl.disableAllBackendNodes();
-        stopBackendNode(DC.SECOND, 0);
-        disableLoadBalancerNode(DC.SECOND, 0);
 
         log.infof("Auth servers stopped successfully. Going to stop all cache servers");
 
-        suiteContext.getCacheServersInfo().stream()
+        DC.validDcsStream().forEach(CrossDCTestEnricher::stopCacheServer);
-                .filter(containerInfo -> containerInfo.isStarted())
-                .forEach(containerInfo -> {
-                    stopCacheServer(containerInfo);
-                });
-
         log.infof("Cache servers stopped successfully");
     }
 
-
     @Test
     public void sessionsPreloadTest() throws Exception {
         int sessionsBefore = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME).size();
@@ -75,7 +68,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         List<OAuthClient.AccessTokenResponse> tokenResponses = createInitialSessions(false);
 
         // Start 2nd DC.
-        startBackendNode(DC.SECOND, 0);
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
         // Ensure sessions are loaded in both 1st DC and 2nd DC
@@ -113,15 +106,14 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         stopAllCacheServersAndAuthServers();
 
         // Start cache containers on both DC1 and DC2
-        startCacheServer(DC.FIRST);
+        DC.validDcsStream().forEach(CrossDCTestEnricher::startCacheServer);
-        startCacheServer(DC.SECOND);
 
         // Start Keycloak on DC1. Sessions should be preloaded from DB
-        startBackendNode(DC.FIRST, 0);
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.FIRST, 0);
         enableLoadBalancerNode(DC.FIRST, 0);
 
         // Start Keycloak on DC2. Sessions should be preloaded from remoteCache
-        startBackendNode(DC.SECOND, 0);
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
         // Ensure sessions are loaded in both 1st DC and 2nd DC
@@ -167,9 +159,10 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         }
 
         // Start 2nd DC.
-        startBackendNode(DC.SECOND, 0);
+        CrossDCTestEnricher.startAuthServerBackendNode(DC.SECOND, 0);
         enableLoadBalancerNode(DC.SECOND, 0);
 
+        Retry.execute(() -> {
             // Ensure loginFailures are loaded in both 1st DC and 2nd DC
             int size1 = getTestingClientForStartedNodeInDc(0).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size();
             int size2 = getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME).size();
@@ -180,6 +173,7 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
             Assert.assertEquals(size2, 1);
             Assert.assertEquals(loginFailures1, loginFailuresBefore + SESSIONS_COUNT);
             Assert.assertEquals(loginFailures2, loginFailuresBefore + SESSIONS_COUNT);
+        }, 3, 400);
 
         // On DC2 sessions were preloaded from from remoteCache
         Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::loginFailures"));

testsuite/integration-arquillian/tests/pom.xml

@@ -371,7 +371,7 @@
                     <artifactId>maven-antrun-plugin</artifactId>
                     <executions>
                         <execution>
-                            <id>clean-second-cache-server-arquillian-bug-workaround</id>
+                            <id>clean-second-cache-server-arquillian-bug-workaround</id><!--https://issues.jboss.org/browse/WFARQ-44-->
                             <phase>process-test-resources</phase>
                             <goals><goal>run</goal></goals>
                             <configuration>

travis-run-tests.sh

@@ -74,7 +74,7 @@ if [ $1 == "server-group4" ]; then
     run-server-tests org.keycloak.testsuite.k*.**.*Test,org.keycloak.testsuite.m*.**.*Test,org.keycloak.testsuite.o*.**.*Test,org.keycloak.testsuite.s*.**.*Test
 fi
 
-if [ $1 == "crossdc1" ]; then
+if [ $1 == "crossdc-server" ]; then
     cd testsuite/integration-arquillian
     mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan -DskipTests
 
@@ -84,7 +84,7 @@ if [ $1 == "crossdc1" ]; then
     exit ${PIPESTATUS[0]}
 fi
 
-if [ $1 == "crossdc2" ]; then
+if [ $1 == "crossdc-adapter" ]; then
     cd testsuite/integration-arquillian
     mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan,app-server-wildfly -DskipTests