KEYCLOAK-4407 Ability to restart arquillian containers from test

Co-Authored-By: Hynek Mlnarik <hmlnarik@redhat.com>
KEYCLOAK-4407 Fix connection error if underlying container restarts (63b9da857a8174a0b5e65e70c47ef2e2842f4d4e)

.travis.yml

@@ -15,7 +15,8 @@ env:
     - TESTS=server-group3
     - TESTS=server-group4
     - TESTS=old
-    - TESTS=crossdc
+    - TESTS=crossdc1
+    - TESTS=crossdc2
 
 jdk:
   - oraclejdk8

integration/admin-client/src/main/java/org/keycloak/admin/client/Keycloak.java

@@ -145,4 +145,11 @@ public class Keycloak {
     public void close() {
         client.close();
     }
+
+    /**
+     * @return true if the underlying client is closed.
+     */
+    public boolean isClosed() {
+        return client.isClosed();
+    }
 }

testsuite/integration-arquillian/pom.xml

@@ -42,7 +42,12 @@
         <app.server>undertow</app.server>
 
         <!--component versions-->
-        <!--to update arquillian-core to 1.3.0.Final or higher see https://issues.jboss.org/browse/ARQ-2181 -->
+        <!--
+            to update arquillian-core to 1.3.0.Final or higher
+                - see https://issues.jboss.org/browse/ARQ-2181 
+                - update org.keycloak.testsuite.arquillian.containers.KeycloakContainerTestExtension according to 
+                  current version of org.jboss.arquillian.container.test.impl.ContainerTestExtension
+        -->
         <arquillian-core.version>1.2.1.Final</arquillian-core.version>
         <!--the version of shrinkwrap_resolver should align with the version in arquillian-bom-->
         <shrinkwrap-resolver.version>2.2.6</shrinkwrap-resolver.version>

testsuite/integration-arquillian/tests/base/pom.xml

@@ -670,6 +670,7 @@
             <id>auth-servers-crossdc-undertow</id>
             <properties>
                 <skip.clean.second.cache>false</skip.clean.second.cache>
+                <exclude.crossdc>-</exclude.crossdc>
             </properties>
         </profile>
         <profile>
@@ -677,6 +678,7 @@
             <properties>
                 <skip.clean.second.cache>false</skip.clean.second.cache>
                 <skip.copy.auth.crossdc.nodes>false</skip.copy.auth.crossdc.nodes>
+                <exclude.crossdc>-</exclude.crossdc>
             </properties>
         </profile>
         <profile>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/ProfileAssume.java

@@ -54,7 +54,7 @@ public class ProfileAssume {
     }
 
     public static void assumeFeatureEnabled(Profile.Feature feature) {
-        Assume.assumeTrue("Ignoring test as " + feature.name() + " is not enabled", isFeatureEnabled(feature));
+        Assume.assumeTrue("Ignoring test as feature " + feature.name() + " is not enabled", isFeatureEnabled(feature));
     }
 
     public static void assumePreview() {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AppServerTestEnricher.java

@@ -23,6 +23,7 @@ import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.core.api.Instance;
 import org.jboss.arquillian.core.api.annotation.Inject;
 import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.AfterClass;
 import org.jboss.arquillian.test.spi.event.suite.BeforeClass;
 import org.jboss.logging.Logger;
 import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
@@ -63,9 +64,9 @@ public class AppServerTestEnricher {
         Class<?> annotatedClass = getNearestSuperclassWithAppServerAnnotation(testClass);
 
         if (annotatedClass == null) return null; // no @AppServerContainer annotation --> no adapter test
-        
+
         AppServerContainer[] appServerContainers = annotatedClass.getAnnotationsByType(AppServerContainer.class);
-        
+
         List<String> appServerQualifiers = new ArrayList<>();
         for (AppServerContainer appServerContainer : appServerContainers) {
             appServerQualifiers.add(appServerContainer.value());
@@ -87,7 +88,7 @@ public class AppServerTestEnricher {
 
         return String.format("%s://%s:%s", scheme, host, port + clusterPortOffset);
     }
-    
+
     private static int parsePort(String property) {
         try {
             return Integer.parseInt(System.getProperty(property));
@@ -182,6 +183,19 @@ public class AppServerTestEnricher {
         }
     }
 
+    public void stopAppServer(@Observes(precedence = 1) AfterClass event) {
+        if (testContext.getAppServerInfo() == null) {
+            return; // no adapter test
+        }
+
+        ContainerController controller = containerConrollerInstance.get();
+
+        if (controller.isStarted(testContext.getAppServerInfo().getQualifier())) {
+            log.info("Stopping app server: " + testContext.getAppServerInfo().getQualifier());
+            controller.stop(testContext.getAppServerInfo().getQualifier());
+        }
+    }
+
     /**
      * Workaround for WFARQ-44. It cannot be used 'cleanServerBaseDir' property.
      * 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -16,14 +16,15 @@
  */
 package org.keycloak.testsuite.arquillian;
 
-import org.jboss.arquillian.container.spi.Container;
 import org.jboss.arquillian.container.spi.ContainerRegistry;
 import org.jboss.arquillian.container.spi.event.StartContainer;
 import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
 import org.jboss.arquillian.container.spi.event.StopContainer;
+import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.core.api.Event;
 import org.jboss.arquillian.core.api.Instance;
 import org.jboss.arquillian.core.api.InstanceProducer;
+import org.jboss.arquillian.core.api.annotation.ApplicationScoped;
 import org.jboss.arquillian.core.api.annotation.Inject;
 import org.jboss.arquillian.core.api.annotation.Observes;
 import org.jboss.arquillian.test.spi.annotation.ClassScoped;
@@ -50,7 +51,6 @@ import java.util.Set;
 
 import java.util.stream.Collectors;
 import javax.ws.rs.NotFoundException;
-import org.jboss.arquillian.core.api.annotation.ApplicationScoped;
 
 /**
  *
@@ -61,6 +61,8 @@ public class AuthServerTestEnricher {
 
     protected static final Logger log = Logger.getLogger(AuthServerTestEnricher.class);
 
+    @Inject 
+    private Instance<ContainerController> containerConroller;
     @Inject
     private Instance<ContainerRegistry> containerRegistry;
 
@@ -89,10 +91,6 @@ public class AuthServerTestEnricher {
     public static final Boolean START_MIGRATION_CONTAINER = "auto".equals(System.getProperty("migration.mode")) ||
             "manual".equals(System.getProperty("migration.mode"));
 
-    // In manual mode are all containers despite loadbalancers started in mode "manual" and nothing is managed through "suite".
-    // Useful for tests, which require restart servers etc.
-    public static final String MANUAL_MODE = "manual.mode";
-
     @Inject
     @SuiteScoped
     private InstanceProducer<SuiteContext> suiteContextProducer;
@@ -123,9 +121,8 @@ public class AuthServerTestEnricher {
     }
 
     public static OnlineManagementClient getManagementClient() {
-        OnlineManagementClient managementClient;
         try {
-            managementClient = ManagementClient.online(OnlineOptions
+            return ManagementClient.online(OnlineOptions
                     .standalone()
                     .hostAndPort(System.getProperty("auth.server.host", "localhost"), Integer.parseInt(System.getProperty("auth.server.management.port", "10090")))
                     .build()
@@ -133,11 +130,8 @@ public class AuthServerTestEnricher {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-
-
-        return managementClient;
     }
-    
+
     public void distinguishContainersInConsoleOutput(@Observes(precedence = 5) StartContainer event) {
         log.info("************************" + event.getContainer().getName()
                 + "*****************************************************************************");
@@ -148,21 +142,18 @@ public class AuthServerTestEnricher {
           .map(ContainerInfo::new)
           .collect(Collectors.toSet());
 
-        // A way to specify that containers should be in mode "manual" rather then "suite"
-        checkManualMode(containers);
-
         suiteContext = new SuiteContext(containers);
 
         if (AUTH_SERVER_CROSS_DC) {
             // if cross-dc mode enabled, load-balancer is the frontend of datacenter cluster
             containers.stream()
-              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
+                .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
-              .forEach(c -> {
+                .forEach(c -> {
-                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                    String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
-                String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
+                    String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
-                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                    updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
-                suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
+                    suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
-              });
+                });
 
             if (suiteContext.getDcAuthServerInfo().isEmpty()) {
                 throw new IllegalStateException("Not found frontend container (load balancer): " + AUTH_SERVER_BALANCER);
@@ -181,7 +172,7 @@ public class AuthServerTestEnricher {
                         String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
                         suiteContext.addAuthServerBackendsInfo(Integer.valueOf(dcString), c);
                     });
-            
+
             containers.stream()
                     .filter(c -> c.getQualifier().startsWith("cache-server-cross-dc-"))
                     .sorted((a, b) -> a.getQualifier().compareTo(b.getQualifier()))
@@ -258,6 +249,7 @@ public class AuthServerTestEnricher {
         }
 
         suiteContextProducer.set(suiteContext);
+        CacheServerTestEnricher.initializeSuiteContext(suiteContext);
         log.info("\n\n" + suiteContext);
     }
 
@@ -295,6 +287,12 @@ public class AuthServerTestEnricher {
         }
     }
 
+    public void startAuthContainer(@Observes(precedence = 0) StartSuiteContainers event) {
+        //frontend-only (either load-balancer or auth-server)
+        log.debug("Starting auth server before suite");
+        startContainerEvent.fire(new StartContainer(suiteContext.getAuthServerInfo().getArquillianContainer()));
+    }
+
     public void checkServerLogs(@Observes(precedence = -1) BeforeSuite event) throws IOException, InterruptedException {
         boolean checkLog = Boolean.parseBoolean(System.getProperty("auth.server.log.check", "true"));
         if (checkLog && suiteContext.getAuthServerInfo().isJBossBased()) {
@@ -316,6 +314,13 @@ public class AuthServerTestEnricher {
     }
 
     public void afterClass(@Observes(precedence = 2) AfterClass event) {
+        //check if a test accidentally left the auth-server not running
+        ContainerController controller = containerConroller.get();
+        if (!controller.isStarted(suiteContext.getAuthServerInfo().getQualifier())) {
+            log.warn("Auth server wasn't running. Starting " + suiteContext.getAuthServerInfo().getQualifier());
+            controller.start(suiteContext.getAuthServerInfo().getQualifier());
+        }
+
         TestContext testContext = testContextProducer.get();
 
         Keycloak adminClient = testContext.getAdminClient();
@@ -335,32 +340,18 @@ public class AuthServerTestEnricher {
 
     public static void removeTestRealms(TestContext testContext, Keycloak adminClient) {
         List<RealmRepresentation> testRealmReps = testContext.getTestRealmReps();
-        if (testRealmReps != null) {
+        if (testRealmReps != null && !testRealmReps.isEmpty()) {
             log.info("removing test realms after test class");
+            StringBuilder realms = new StringBuilder();
             for (RealmRepresentation testRealm : testRealmReps) {
-                String realmName = testRealm.getRealm();
-                log.info("removing realm: " + realmName);
                 try {
-                    adminClient.realms().realm(realmName).remove();
+                    adminClient.realms().realm(testRealm.getRealm()).remove();
+                    realms.append(testRealm.getRealm()).append(", ");
                 } catch (NotFoundException e) {
                     // Ignore
                 }
             }
-        }
+            log.info("removed realms: " + realms);
-    }
-
-
-    private void checkManualMode(Set<ContainerInfo> containers) {
-        String manualMode = System.getProperty(MANUAL_MODE);
-
-        if (Boolean.parseBoolean(manualMode)) {
-
-            containers.stream()
-                    .filter(containerInfo -> !containerInfo.getQualifier().contains("balancer"))
-                    .forEach(containerInfo -> {
-                        log.infof("Container '%s' will be in manual mode", containerInfo.getQualifier());
-                        containerInfo.getArquillianContainer().getContainerConfiguration().setMode("manual");
-                    });
         }
     }
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheServerTestEnricher.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2018 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.spi.Validate;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.AfterClass;
+import org.jboss.logging.Logger;
+
+import org.keycloak.testsuite.crossdc.DC;
+
+/**
+ *
+ * @author vramik
+ */
+public class CacheServerTestEnricher {
+
+    protected static final Logger log = Logger.getLogger(CacheServerTestEnricher.class);
+    private static SuiteContext suiteContext;
+
+    @Inject 
+    private Instance<ContainerController> containerController;
+
+    static void initializeSuiteContext(SuiteContext suiteContext) {
+        Validate.notNull(suiteContext, "Suite context cannot be null.");
+        CacheServerTestEnricher.suiteContext = suiteContext;
+    }
+
+    public void afterClass(@Observes(precedence = 4) AfterClass event) {
+        if (!suiteContext.getCacheServersInfo().isEmpty()) {
+            stopCacheServer(suiteContext.getCacheServersInfo().get(DC.FIRST.ordinal()));
+            stopCacheServer(suiteContext.getCacheServersInfo().get(DC.SECOND.ordinal()));
+        }
+    }
+
+    private void stopCacheServer(ContainerInfo cacheServer) {
+        if (containerController.get().isStarted(cacheServer.getQualifier())) {
+            log.infof("Stopping %s", cacheServer.getQualifier());
+
+            containerController.get().stop(cacheServer.getQualifier());
+
+            // Workaround for possible arquillian bug. Needs to cleanup dir manually
+            String setupCleanServerBaseDir = getContainerProperty(cacheServer, "setupCleanServerBaseDir");
+            String cleanServerBaseDir = getContainerProperty(cacheServer, "cleanServerBaseDir");
+
+            if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
+                log.infof("Going to clean directory: %s", cleanServerBaseDir);
+
+                File dir = new File(cleanServerBaseDir);
+                if (dir.exists()) {
+                    try {
+                        dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));
+
+                        File deploymentsDir = new File(dir, "deployments");
+                        FileUtils.forceMkdir(deploymentsDir);
+                    } catch (IOException ioe) {
+                        throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
+                    }
+                }
+            }
+
+            log.infof("Stopped %s", cacheServer.getQualifier());
+        }
+    }
+
+    private String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
+        return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/CacheStatisticsControllerEnricher.java

@@ -95,7 +95,9 @@ public class CacheStatisticsControllerEnricher implements TestEnricher {
 
         if (annotation.domain().isEmpty()) {
             try {
-                Retry.execute(() -> value.reset(), 2, 150);
+                LOG.debug("Going to try reset InfinispanCacheStatistics (2 attempts, 150 ms interval)");
+                int execute = Retry.execute(() -> value.reset(), 2, 150);
+                LOG.debug("reset in " + execute + " attempts");
             } catch (RuntimeException ex) {
                 if (annotation.dc() != DC.UNDEFINED && annotation.dcNodeIndex() != -1
                    && suiteContext.get().getAuthServerBackendsInfo(annotation.dc().getDcIndex()).get(annotation.dcNodeIndex()).isStarted()) {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/DeploymentTargetModifier.java

@@ -59,7 +59,7 @@ public class DeploymentTargetModifier extends AnnotationDeploymentScenarioGenera
 
         List<DeploymentDescription> deployments = super.generate(testClass);
 
-        checkTestDeployments(deployments, testClass);
+        checkTestDeployments(deployments, testClass, context.isAdapterTest());
         List<String> appServerQualifiers = getAppServerQualifiers(testClass.getJavaClass());
         if (appServerQualifiers == null) return deployments; // no adapter test
 
@@ -87,11 +87,11 @@ public class DeploymentTargetModifier extends AnnotationDeploymentScenarioGenera
         return deployments;
     }
 
-    private void checkTestDeployments(List<DeploymentDescription> descriptions, TestClass testClass) {
+    private void checkTestDeployments(List<DeploymentDescription> descriptions, TestClass testClass, boolean isAdapterTest) {
         for (DeploymentDescription deployment : descriptions) {
             if (deployment.getTarget() != null) {
                 String containerQualifier = deployment.getTarget().getName();
-                if (AUTH_SERVER_CURRENT.equals(containerQualifier)) {
+                if (AUTH_SERVER_CURRENT.equals(containerQualifier) || (!isAdapterTest && "_DEFAULT_".equals(containerQualifier))) {
                     String newAuthServerQualifier = AuthServerTestEnricher.AUTH_SERVER_CONTAINER;
                     updateServerQualifier(deployment, testClass, newAuthServerQualifier);
                 } else if (containerQualifier.contains(APP_SERVER_CURRENT)) {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -66,6 +66,7 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .observer(JmxConnectorRegistryCreator.class)
                 .observer(AuthServerTestEnricher.class)
                 .observer(AppServerTestEnricher.class)
+                .observer(CacheServerTestEnricher.class)
                 .observer(H2TestEnricher.class);
         builder
                 .service(TestExecutionDecider.class, MigrationTestExecutionDecider.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/TestContext.java

@@ -49,7 +49,7 @@ public final class TestContext {
 
     private Keycloak adminClient;
     private KeycloakTestingClient testingClient;
-    private List<RealmRepresentation> testRealmReps;
+    private List<RealmRepresentation> testRealmReps = new ArrayList<>();
 
     // Track if particular test was initialized. What exactly means "initialized" is test dependent (Eg. some user in @Before method was created, so we can set initialized to true
     // to avoid creating user when @Before method is executed for 2nd time)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakContainerEventsController.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2018 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.containers;
+
+import java.util.List;
+import org.jboss.arquillian.container.impl.client.container.ContainerDeployController;
+import org.jboss.arquillian.container.spi.Container;
+import org.jboss.arquillian.container.spi.ContainerRegistry;
+import org.jboss.arquillian.container.spi.client.deployment.Deployment;
+import org.jboss.arquillian.container.spi.client.deployment.DeploymentScenario;
+import org.jboss.arquillian.container.spi.event.DeploymentEvent;
+import org.jboss.arquillian.container.spi.event.DeployDeployment;
+import org.jboss.arquillian.container.spi.event.DeployManagedDeployments;
+
+import org.jboss.arquillian.container.spi.event.ContainerMultiControlEvent;
+import org.jboss.arquillian.container.spi.event.StopClassContainers;
+import org.jboss.arquillian.container.spi.event.StopManualContainers;
+import org.jboss.arquillian.container.spi.event.StopSuiteContainers;
+import org.jboss.arquillian.container.spi.event.UnDeployManagedDeployments;
+import org.jboss.arquillian.container.test.impl.client.ContainerEventController;
+import org.jboss.arquillian.core.api.Event;
+import org.jboss.arquillian.core.api.Injector;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.spi.event.suite.AfterClass;
+import org.jboss.arquillian.test.spi.event.suite.AfterSuite;
+
+/**
+ * Changes behaviour of original ContainerEventController to stop manual containers 
+ * @AfterSuite, not @AfterClass
+ * 
+ * @see https://issues.jboss.org/browse/ARQ-2186
+ * 
+ * @author vramik
+ */
+public class KeycloakContainerEventsController extends ContainerEventController {
+
+    @Inject
+    private Event<ContainerMultiControlEvent> container;
+
+    @Override
+    public void execute(@Observes AfterSuite event) {
+        container.fire(new StopManualContainers());
+        container.fire(new StopSuiteContainers());
+    }
+
+    @Override
+    public void execute(@Observes(precedence = 3) AfterClass event) {
+        try {
+            container.fire(new UnDeployManagedDeployments());
+        } finally {
+            container.fire(new StopClassContainers());
+        }
+    }
+
+
+    /*
+     * Coppied from org.jboss.arquillian.container.impl.client.container.ContainerDeployController
+     * 
+     * Overrides a condition that container cannot be in manual mode, and deploys the deployment
+     * if the container is started
+     */
+    @Inject
+    private Instance<Injector> injector;
+    @Inject
+    private Instance<DeploymentScenario> deploymentScenario;
+    @Inject
+    private Instance<ContainerRegistry> containerRegistry;
+
+    public void deployManaged(@Observes DeployManagedDeployments event) throws Exception {
+        forEachManagedDeployment(new ContainerDeployController.Operation<Container, Deployment>() {
+            @Inject
+            private Event<DeploymentEvent> event;
+
+            @Override
+            public void perform(Container container, Deployment deployment) throws Exception {
+                if (container.getState().equals(Container.State.STARTED)) {
+                    event.fire(new DeployDeployment(container, deployment));
+                }
+            }
+        });
+    }
+
+    private void forEachManagedDeployment(ContainerDeployController.Operation<Container, Deployment> operation) throws Exception {
+        DeploymentScenario scenario = this.deploymentScenario.get();
+        if (scenario == null) {
+            return;
+        }
+        forEachDeployment(scenario.managedDeploymentsInDeployOrder(), operation);
+    }
+
+    private void forEachDeployment(List<Deployment> deployments, ContainerDeployController.Operation<Container, Deployment> operation)
+        throws Exception {
+        injector.get().inject(operation);
+        ContainerRegistry containerRegistry = this.containerRegistry.get();
+        if (containerRegistry == null) {
+            return;
+        }
+        for (Deployment deployment : deployments) {
+            Container container = containerRegistry.getContainer(deployment.getDescription().getTarget());
+            operation.perform(container, deployment);
+        }
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakContainerTestExtension.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2018 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.containers;
+
+import org.jboss.arquillian.container.test.impl.ClientTestInstanceEnricher;
+import org.jboss.arquillian.container.test.impl.client.LocalCommandService;
+import org.jboss.arquillian.container.test.impl.client.container.ClientContainerControllerCreator;
+import org.jboss.arquillian.container.test.impl.client.container.ContainerRestarter;
+import org.jboss.arquillian.container.test.impl.client.container.command.ContainerCommandObserver;
+import org.jboss.arquillian.container.test.impl.client.deployment.ClientDeployerCreator;
+import org.jboss.arquillian.container.test.impl.client.deployment.DeploymentGenerator;
+import org.jboss.arquillian.container.test.impl.client.deployment.command.DeploymentCommandObserver;
+import org.jboss.arquillian.container.test.impl.client.deployment.tool.ArchiveDeploymentToolingExporter;
+import org.jboss.arquillian.container.test.impl.client.protocol.ProtocolRegistryCreator;
+import org.jboss.arquillian.container.test.impl.client.protocol.local.LocalProtocol;
+import org.jboss.arquillian.container.test.impl.deployment.ArquillianDeploymentAppender;
+import org.jboss.arquillian.container.test.impl.enricher.resource.ContainerControllerProvider;
+import org.jboss.arquillian.container.test.impl.enricher.resource.DeployerProvider;
+import org.jboss.arquillian.container.test.impl.enricher.resource.InitialContextProvider;
+import org.jboss.arquillian.container.test.impl.enricher.resource.RemoteResourceCommandObserver;
+import org.jboss.arquillian.container.test.impl.enricher.resource.URIResourceProvider;
+import org.jboss.arquillian.container.test.impl.enricher.resource.URLResourceProvider;
+import org.jboss.arquillian.container.test.impl.execution.ClientBeforeAfterLifecycleEventExecuter;
+import org.jboss.arquillian.container.test.impl.execution.ClientTestExecuter;
+import org.jboss.arquillian.container.test.impl.execution.LocalTestExecuter;
+import org.jboss.arquillian.container.test.impl.execution.RemoteTestExecuter;
+import org.jboss.arquillian.container.test.spi.client.deployment.AuxiliaryArchiveAppender;
+import org.jboss.arquillian.container.test.spi.client.protocol.Protocol;
+import org.jboss.arquillian.container.test.spi.command.CommandService;
+import org.jboss.arquillian.core.spi.LoadableExtension;
+import org.jboss.arquillian.test.impl.TestContextHandler;
+import org.jboss.arquillian.test.impl.context.ClassContextImpl;
+import org.jboss.arquillian.test.impl.context.SuiteContextImpl;
+import org.jboss.arquillian.test.impl.context.TestContextImpl;
+import org.jboss.arquillian.test.impl.enricher.resource.ArquillianResourceTestEnricher;
+import org.jboss.arquillian.test.spi.TestEnricher;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+
+/**
+ * KeycloakContainerTestExtension
+ * 
+ * This Extension Overrides the original ContainerTestExtension. 
+ * 
+ * Needed to change the behavior of ContainerEventController 
+ * to stopManualContainers @AfterSuite instead of @AfterClass
+ *
+ * @see base/src/main/resources/META-INF/services/org.jboss.arquillian.core.spi.LoadableExtension
+ * @see https://issues.jboss.org/browse/ARQ-2186
+ * 
+ * @author vramik
+ * @version $Revision: $
+ */
+public class KeycloakContainerTestExtension implements LoadableExtension {
+    @Override
+    public void register(ExtensionBuilder builder) {
+        registerOriginal(builder);
+        
+//      Overriden ContainerEventController
+        builder.observer(KeycloakContainerEventsController.class);
+    }
+    
+    private void registerOriginal(ExtensionBuilder builder) {
+        // Start -> Copied from TestExtension
+        builder.context(SuiteContextImpl.class)
+            .context(ClassContextImpl.class)
+            .context(TestContextImpl.class);
+
+        builder.observer(TestContextHandler.class)
+            .observer(ClientTestInstanceEnricher.class);
+
+        // End -> Copied from TestExtension
+
+        builder.service(AuxiliaryArchiveAppender.class, ArquillianDeploymentAppender.class)
+            .service(TestEnricher.class, ArquillianResourceTestEnricher.class)
+            .service(Protocol.class, LocalProtocol.class)
+            .service(CommandService.class, LocalCommandService.class)
+            .service(ResourceProvider.class, URLResourceProvider.class)
+            .service(ResourceProvider.class, URIResourceProvider.class)
+            .service(ResourceProvider.class, DeployerProvider.class)
+            .service(ResourceProvider.class, InitialContextProvider.class)
+            .service(ResourceProvider.class, ContainerControllerProvider.class);
+
+//        ContainerEventController is overriden
+//        builder.observer(ContainerEventController.class)
+          builder.observer(ContainerRestarter.class)
+            .observer(DeploymentGenerator.class)
+            .observer(ArchiveDeploymentToolingExporter.class)
+            .observer(ProtocolRegistryCreator.class)
+            .observer(ClientContainerControllerCreator.class)
+            .observer(ClientDeployerCreator.class)
+            .observer(ClientBeforeAfterLifecycleEventExecuter.class)
+            .observer(ClientTestExecuter.class)
+            .observer(LocalTestExecuter.class)
+            .observer(RemoteTestExecuter.class)
+            .observer(DeploymentCommandObserver.class)
+            .observer(ContainerCommandObserver.class)
+            .observer(RemoteResourceCommandObserver.class);
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/h2/H2TestEnricher.java

@@ -21,7 +21,7 @@ public class H2TestEnricher {
 
     private Server server = null;
 
-    public void startH2(@Observes(precedence = 2) BeforeSuite event) throws SQLException {
+    public void startH2(@Observes(precedence = 3) BeforeSuite event) throws SQLException {
         if (runH2) {
             log.info("Starting H2 database.");
             server = Server.createTcpServer();

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/jmx/JmxConnectorRegistryCreator.java

@@ -19,6 +19,7 @@ package org.keycloak.testsuite.arquillian.jmx;
 import java.io.IOException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.logging.Level;
 import javax.management.remote.JMXConnector;
 import javax.management.remote.JMXConnectorFactory;
 import javax.management.remote.JMXServiceURL;
@@ -47,24 +48,28 @@ public class JmxConnectorRegistryCreator {
 
                 private volatile ConcurrentMap<JMXServiceURL, JMXConnector> connectors = new ConcurrentHashMap<>();
 
+                private JMXConnector createConnection(JMXServiceURL key) {
+                    try {
+                        final JMXConnector conn = JMXConnectorFactory.newJMXConnector(key, null);
+                        conn.connect();
+                        log.infof("Connected to JMX Service URL: %s", key);
+                        return conn;
+                    } catch (IOException ex) {
+                        throw new RuntimeException("Could not instantiate JMX connector for " + key, ex);
+                    }
+                }
+
                 @Override
                 public JMXConnector getConnection(JMXServiceURL url) {
                     
-                    JMXConnector res = connectors.get(url);
+                    JMXConnector res = connectors.computeIfAbsent(url, this::createConnection);
-                    if (res == null) {
+                    // Check connection is alive
-                        try {
+                    try {
-                            final JMXConnector conn = JMXConnectorFactory.newJMXConnector(url, null);
+                        res.getMBeanServerConnection().getMBeanCount();
-                            res = connectors.putIfAbsent(url, conn);
+                    } catch (IOException ex) {
-                            if (res == null) {
+                        // retry in case connection is not alive
-                                res = conn;
+                        try { res.close(); } catch (IOException e) { }
-                            }
+                        connectors.replace(url, createConnection(url));
-                            res.connect();
-                            log.infof("Connected to JMX Service URL: %s", url);
-                        } catch (IOException ex) {
-                            //remove conn from connectors in case something goes wrong. The connection will be established on-demand
-                            connectors.remove(url, res);
-                            throw new RuntimeException("Could not instantiate JMX connector for " + url, ex);
-                        }
                     }
                     return res;
                 }

testsuite/integration-arquillian/tests/base/src/main/resources/META-INF/services/org.jboss.arquillian.core.spi.LoadableExtension

@@ -16,5 +16,9 @@
 #
 
 org.keycloak.testsuite.arquillian.KeycloakArquillianExtension
+
 !org.jboss.arquillian.container.impl.ContainerExtension
-org.keycloak.testsuite.arquillian.containers.MultipleContainersExtension
+org.keycloak.testsuite.arquillian.containers.MultipleContainersExtension
+
+!org.jboss.arquillian.container.test.impl.ContainerTestExtension
+org.keycloak.testsuite.arquillian.containers.KeycloakContainerTestExtension

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractKeycloakTest.java

@@ -29,7 +29,6 @@ import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.admin.client.resource.AuthenticationManagementResource;
-import org.keycloak.admin.client.resource.RealmResource;
 import org.keycloak.admin.client.resource.RealmsResource;
 import org.keycloak.admin.client.resource.UserResource;
 import org.keycloak.admin.client.resource.UsersResource;
@@ -149,7 +148,7 @@ public abstract class AbstractKeycloakTest {
     @Before
     public void beforeAbstractKeycloakTest() throws Exception {
         adminClient = testContext.getAdminClient();
-        if (adminClient == null) {
+        if (adminClient == null || adminClient.isClosed()) {
             String authServerContextRoot = suiteContext.getAuthServerInfo().getContextRoot().toString();
             adminClient = AdminClientUtil.createAdminClient(suiteContext.isAdapterCompatTesting(), authServerContextRoot);
             testContext.setAdminClient(adminClient);
@@ -170,7 +169,7 @@ public abstract class AbstractKeycloakTest {
 
         beforeAbstractKeycloakTestRealmImport();
 
-        if (testContext.getTestRealmReps() == null) {
+        if (testContext.getTestRealmReps().isEmpty()) {
             importTestRealms();
 
             if (!isImportAfterEachMethod()) {
@@ -184,6 +183,8 @@ public abstract class AbstractKeycloakTest {
 
     protected void beforeAbstractKeycloakTestRealmImport() throws Exception {
     }
+    protected void postAfterAbstractKeycloak() {
+    }
 
     @After
     public void afterAbstractKeycloakTest() {
@@ -216,6 +217,8 @@ public abstract class AbstractKeycloakTest {
             testContext.getCleanups().clear();
         }
 
+        postAfterAbstractKeycloak();
+
         // Remove all browsers from queue
         DroneUtils.resetQueue();
     }
@@ -316,12 +319,11 @@ public abstract class AbstractKeycloakTest {
     }
 
     public void importRealm(RealmRepresentation realm) {
-        log.debug("importing realm: " + realm.getRealm());
+        log.debug("--importing realm: " + realm.getRealm());
-        try { // TODO - figure out a way how to do this without try-catch
+        try {
-            RealmResource realmResource = adminClient.realms().realm(realm.getRealm());
+            adminClient.realms().realm(realm.getRealm()).remove();
-            log.debug("realm already exists on server, re-importing");
+            log.debug("realm already existed on server, re-importing");
-            realmResource.remove();
+        } catch (NotFoundException ignore) {
-        } catch (NotFoundException nfe) {
             // expected when realm does not exist
         }
         adminClient.realms().create(realm);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AbstractTestRealmKeycloakTest.java

@@ -65,6 +65,7 @@ public abstract class AbstractTestRealmKeycloakTest extends AbstractKeycloakTest
 
     @Override
     public void addTestRealms(List<RealmRepresentation> testRealms) {
+        log.debug("Adding test realm for import from testrealm.json");
         RealmRepresentation testRealm = loadJson(getClass().getResourceAsStream("/testrealm.json"), RealmRepresentation.class);
 
         testRealms.add(testRealm);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/adapter/servlet/DemoServletsAdapterTest.java

@@ -648,6 +648,8 @@ public class DemoServletsAdapterTest extends AbstractServletsAdapterTest {
 
     @Test
     public void testVersion() {
+        jsDriver.navigate().to(suiteContext.getAuthServerInfo().getContextRoot().toString() + "/auth");
+        WaitUtils.waitForPageToLoad();
         jsDriver.navigate().to(suiteContext.getAuthServerInfo().getContextRoot().toString() +
                 "/auth/admin/master/console/#/server-info");
         WaitUtils.waitForPageToLoad();

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java

@@ -32,6 +32,7 @@ import java.util.function.Function;
 import org.hamcrest.Matcher;
 import org.junit.Before;
 import static org.junit.Assert.assertThat;
+import org.keycloak.representations.idm.ClientRepresentation;
 
 /**
  *
@@ -47,13 +48,17 @@ public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
 
     @Override
     public void configureTestRealm(RealmRepresentation testRealm) {
-        findTestApp(testRealm).setDirectAccessGrantsEnabled(true);
+        log.debug("Configuring test realm '" + testRealm.getRealm() + "'. Enabling direct access grant.");
+        ClientRepresentation testApp = findTestApp(testRealm);
+        if (testApp == null) {
+            throw new IllegalStateException("Couldn't find the 'test-app' within the realm '" + testRealm.getRealm() + "'");
+        }
+        testApp.setDirectAccessGrantsEnabled(true);
     }
 
-
-
     @Override
     public void addTestRealms(List<RealmRepresentation> testRealms) {
+        log.debug("--DC: AbstractAdminCrossDCTest.addTestRealms - adding realm: " + REALM_NAME);
         super.addTestRealms(testRealms);
 
         RealmRepresentation adminRealmRep = new RealmRepresentation();
@@ -76,6 +81,7 @@ public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
 
     @Before
     public void setRealm() {
+        log.debug("--DC: AbstractAdminCrossDCTest.setRealm");
         realm = adminClient.realm(REALM_NAME);
         realmId = realm.toRepresentation().getId();
     }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java

@@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.models.Constants;
 import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
+import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
 import org.keycloak.testsuite.arquillian.ContainerInfo;
 import org.keycloak.testsuite.arquillian.LoadBalancerController;
 import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
@@ -27,11 +28,11 @@ import org.keycloak.testsuite.auth.page.AuthRealm;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 import org.jboss.arquillian.container.test.api.ContainerController;
 import org.jboss.arquillian.test.api.ArquillianResource;
 import org.junit.After;
@@ -41,6 +42,7 @@ import org.keycloak.testsuite.client.KeycloakTestingClient;
 import static org.hamcrest.Matchers.lessThan;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import org.keycloak.representations.idm.RealmRepresentation;
 
 /**
  * Abstract cross-data-centre test that defines primitives for handling cross-DC setup.
@@ -64,100 +66,108 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
 
     protected Map<ContainerInfo, KeycloakTestingClient> backendTestingClients = new HashMap<>();
 
+    @Before
+    @Override
+    public void beforeAbstractKeycloakTest() throws Exception {
+        log.debug("--DC: Starting both cache servers and first node from each DC");
+        startCacheServer(DC.FIRST);
+        startCacheServer(DC.SECOND);
+
+        startBackendNode(DC.FIRST, 0);
+        startBackendNode(DC.SECOND, 0);
+
+        initRESTClientsForStartedNodes();
+        suspendPeriodicTasks();
+
+        enableOnlyFirstNodeInFirstDc();
+
+        super.beforeAbstractKeycloakTest();
+    }
+
+    @Override
+    public void deleteCookies() {
+        //Overrides AbstractTestRealmKeycloakTest.deleteCookies 
+        //as it tries to delete cookies in 'test' realm @After test
+        //when backend containers are stopped already.
+    }
+    
     @After
-    @Before
+    @Override
-    public void enableOnlyFirstNodeInFirstDc() {
+    public void afterAbstractKeycloakTest() {
+        log.debug("--DC: after AbstractCrossDCTest");
+        enableOnlyFirstNodeInFirstDc();
+        
+        super.afterAbstractKeycloakTest();
+        
+        restorePeriodicTasks();
+        removeTestRealms();
+        terminateStartedServers();
+        loadBalancerCtrl.disableAllBackendNodes();
+    }
+
+    private void enableOnlyFirstNodeInFirstDc() {
+        log.debug("--DC: Enable only first node in first datacenter");
         this.loadBalancerCtrl.disableAllBackendNodes();
-        loadBalancerCtrl.enableBackendNodeByName(getAutomaticallyStartedBackendNodes(DC.FIRST)
+        if (!getBackendNode(DC.FIRST, 0).isStarted()) {
-                        .findFirst()
+            throw new IllegalStateException("--DC: Trying to enable not started node on load-balancer");
-                        .orElseThrow(() -> new IllegalStateException("No node is started automatically"))
+        }
-                        .getQualifier()
+        loadBalancerCtrl.enableBackendNodeByName(getBackendNode(DC.FIRST, 0).getQualifier());
-        );
     }
 
-    @Before
+    private void removeTestRealms() {
-    public void terminateManuallyStartedServers() {
+        testContext.getTestRealmReps().stream().forEach((RealmRepresentation realm) -> deleteAllCookiesForRealm(realm.getRealm()));
-        log.debug("Halting all nodes that are started manually");
+
+        log.debug("--DC: removing rest realms");
+        AuthServerTestEnricher.removeTestRealms(testContext, adminClient);
+        testContext.setTestRealmReps(new ArrayList<>());
+    }
+    
+    protected void terminateStartedServers() {
+        log.debug("--DC: Halting all nodes that are started");
         this.suiteContext.getDcAuthServerBackendsInfo().stream()
-          .flatMap(List::stream)
+            .flatMap(List::stream)
-          .filter(ContainerInfo::isStarted)
+            .filter(ContainerInfo::isStarted)
-          .filter(ContainerInfo::isManual)
+            .forEach((ContainerInfo containerInfo) -> {
-          .forEach(containerInfo -> {
+                containerController.stop(containerInfo.getQualifier());
-              containerController.stop(containerInfo.getQualifier());
+                removeRESTClientsForNode(containerInfo);
-              removeRESTClientsForNode(containerInfo);
+            });
-          });
     }
 
-    @Before
+    private void initRESTClientsForStartedNodes() {
-    public void initRESTClientsForStartedNodes() {
+        log.debug("--DC: Init REST clients for started nodes");
-        log.debug("Init REST clients for automatically started nodes");
         this.suiteContext.getDcAuthServerBackendsInfo().stream()
                 .flatMap(List::stream)
                 .filter(ContainerInfo::isStarted)
-                .filter(containerInfo -> !containerInfo.isManual())
                 .forEach(containerInfo -> {
                     createRESTClientsForNode(containerInfo);
                 });
-
     }
 
     // Disable periodic tasks in cross-dc tests. It's needed to have some scenarios more stable.
-    @Before
+    private void suspendPeriodicTasks() {
-    public void suspendPeriodicTasks() {
+        log.debug("--DC: suspendPeriodicTasks");
         backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
             testingClient.testing().suspendPeriodicTasks();
         });
-
     }
 
-    @After
+    private void restorePeriodicTasks() {
-    public void restorePeriodicTasks() {
+        log.debug("--DC: restorePeriodicTasks");
         backendTestingClients.values().stream().forEach((KeycloakTestingClient testingClient) -> {
             testingClient.testing().restorePeriodicTasks();
         });
     }
 
-
-    @Override
-    public void importTestRealms() {
-        enableOnlyFirstNodeInFirstDc();
-        super.importTestRealms();
-    }
-
-    @Override
-    public void afterAbstractKeycloakTest() {
-        enableOnlyFirstNodeInFirstDc();
-        super.afterAbstractKeycloakTest();
-    }
-
-    @Override
-    public void deleteCookies() {
-        enableOnlyFirstNodeInFirstDc();
-        super.deleteCookies();
-    }
-
-    @Before
-    public void initLoadBalancer() {
-        log.debug("Initializing load balancer - only enabling started nodes in the first DC");
-        this.loadBalancerCtrl.disableAllBackendNodes();
-        // Enable only the started nodes in first datacenter
-        this.suiteContext.getDcAuthServerBackendsInfo().get(0).stream()
-          .filter(ContainerInfo::isStarted)
-          .map(ContainerInfo::getQualifier)
-          .forEach(loadBalancerCtrl::enableBackendNodeByName);
-    }
-
     protected Keycloak createAdminClientFor(ContainerInfo node) {
-        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        log.info("--DC: Initializing admin client for " + node.getContextRoot() + "/auth");
         return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
     }
 
     protected KeycloakTestingClient createTestingClientFor(ContainerInfo node) {
-        log.info("Initializing testing client for " + node.getContextRoot() + "/auth");
+        log.info("--DC: Initializing testing client for " + node.getContextRoot() + "/auth");
         return KeycloakTestingClient.getInstance(node.getContextRoot() + "/auth");
     }
 
-
     protected Keycloak getAdminClientForStartedNodeInDc(int dcIndex) {
         ContainerInfo firstStartedNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).stream()
                 .filter(ContainerInfo::isStarted)
@@ -188,7 +198,6 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         return getTestingClientFor(firstStartedNode);
     }
 
-
     /**
      * Get testing client directed to the given node.
      * @param node
@@ -224,14 +233,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         }
     }
 
-
     /**
      * Disables routing requests to the given data center in the load balancer.
      * @param dc
      */
     public void disableDcOnLoadBalancer(DC dc) {
         int dcIndex = dc.ordinal();
-        log.infof("Disabling load balancer for dc=%d", dcIndex);
+        log.infof("--DC: Disabling load balancer for dc=%d", dcIndex);
         this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).forEach(containerInfo -> {
             loadBalancerCtrl.disableBackendNodeByName(containerInfo.getQualifier());
         });
@@ -243,10 +251,10 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      */
     public void enableDcOnLoadBalancer(DC dc) {
         int dcIndex = dc.ordinal();
-        log.infof("Enabling load balancer for dc=%d", dcIndex);
+        log.infof("--DC: Enabling load balancer for dc=%d", dcIndex);
         final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
         if (! dcNodes.stream().anyMatch(ContainerInfo::isStarted)) {
-            log.warnf("No node is started in DC %d", dcIndex);
+            log.warnf("--DC: No node is started in DC %d", dcIndex);
         } else {
             dcNodes.stream()
               .filter(ContainerInfo::isStarted)
@@ -263,7 +271,7 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      */
     public void disableLoadBalancerNode(DC dc, int nodeIndex) {
         int dcIndex = dc.ordinal();
-        log.infof("Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        log.infof("--DC: Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
         loadBalancerCtrl.disableBackendNodeByName(this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex).getQualifier());
     }
 
@@ -274,13 +282,13 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      */
     public void enableLoadBalancerNode(DC dc, int nodeIndex) {
         int dcIndex = dc.ordinal();
-        log.infof("Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        log.infof("--DC: Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
         final ContainerInfo backendNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex);
         if (backendNode == null) {
             throw new IllegalArgumentException("Invalid node with index " + nodeIndex + " for DC " + dcIndex);
         }
         if (! backendNode.isStarted()) {
-            log.warnf("Node %s is not started in DC %d", backendNode.getQualifier(), dcIndex);
+            log.warnf("--DC: Node %s is not started in DC %d", backendNode.getQualifier(), dcIndex);
         }
         loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
     }
@@ -291,19 +299,17 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      * @param nodeIndex
      * @return Started instance descriptor.
      */
-    public ContainerInfo startBackendNode(DC dc, int nodeIndex) {
+    protected ContainerInfo startBackendNode(DC dc, int nodeIndex) {
-        int dcIndex = dc.ordinal();
+        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
-        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
+
-        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
-        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
-        ContainerInfo dcNode = dcNodes.get(nodeIndex);
         assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
-        
-        log.infof("Starting backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dcIndex, nodeIndex);
-        containerController.start(dcNode.getQualifier());
 
-        createRESTClientsForNode(dcNode);
+        if (!containerController.isStarted(dcNode.getQualifier())) {
+            log.infof("--DC: Starting backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
+            containerController.start(dcNode.getQualifier());
 
+            createRESTClientsForNode(dcNode);
+        }
         return dcNode;
     }
 
@@ -313,42 +319,24 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      * @param nodeIndex
      * @return Stopped instance descriptor.
      */
-    public ContainerInfo stopBackendNode(DC dc, int nodeIndex) {
+    protected ContainerInfo stopBackendNode(DC dc, int nodeIndex) {
-        int dcIndex = dc.ordinal();
+        ContainerInfo dcNode = getBackendNode(dc, nodeIndex);
-        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
-        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
-        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
-        ContainerInfo dcNode = dcNodes.get(nodeIndex);
 
         removeRESTClientsForNode(dcNode);
 
         assertTrue("Node " + dcNode.getQualifier() + " has to be controlled manually", dcNode.isManual());
         
-        log.infof("Stopping backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dcIndex, nodeIndex);
+        log.infof("--DC: Stopping backend node: %s (dcIndex: %d, nodeIndex: %d)", dcNode.getQualifier(), dc.ordinal(), nodeIndex);
         containerController.stop(dcNode.getQualifier());
         return dcNode;
     }
 
-    /**
+    private ContainerInfo getBackendNode(DC dc, int nodeIndex) {
-     * Returns stream of all nodes in the given dc that are started manually.
-     * @param dc
-     * @return
-     */
-    public Stream<ContainerInfo> getManuallyStartedBackendNodes(DC dc) {
         int dcIndex = dc.ordinal();
+        assertThat((Integer) dcIndex, lessThan(this.suiteContext.getDcAuthServerBackendsInfo().size()));
         final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
-        return dcNodes.stream().filter(ContainerInfo::isManual);
+        assertThat((Integer) nodeIndex, lessThan(dcNodes.size()));
-    }
+        return dcNodes.get(nodeIndex);
-
-    /**
-     * Returns stream of all nodes in the given dc that are started automatically.
-     * @param dc
-     * @return
-     */
-    public Stream<ContainerInfo> getAutomaticallyStartedBackendNodes(DC dc) {
-        int dcIndex = dc.ordinal();
-        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
-        return dcNodes.stream().filter(c -> ! c.isManual());
     }
 
     /**
@@ -356,14 +344,20 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
      * @param dc
      * @return
      */
-    public ContainerInfo getCacheServer(DC dc) {
+    protected ContainerInfo getCacheServer(DC dc) {
         int dcIndex = dc.ordinal();
         return this.suiteContext.getCacheServersInfo().get(dcIndex);
     }
 
+    protected void startCacheServer(DC dc) {
+        if (!containerController.isStarted(getCacheServer(dc).getQualifier())) {
+            log.infof("--DC: Starting %s", getCacheServer(dc).getQualifier());
+            containerController.start(getCacheServer(dc).getQualifier());
+        }
+    }
 
-    public void stopCacheServer(ContainerInfo cacheServer) {
+    protected void stopCacheServer(ContainerInfo cacheServer) {
-        log.infof("Stopping %s", cacheServer.getQualifier());
+        log.infof("--DC: Stopping %s", cacheServer.getQualifier());
 
         containerController.stop(cacheServer.getQualifier());
 
@@ -372,7 +366,7 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
         String cleanServerBaseDir = cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get("cleanServerBaseDir");
 
         if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
-            log.infof("Going to clean directory: %s", cleanServerBaseDir);
+            log.infof("--DC: Going to clean directory: %s", cleanServerBaseDir);
 
             File dir = new File(cleanServerBaseDir);
             if (dir.exists()) {
@@ -387,10 +381,9 @@ public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest
             }
         }
 
-        log.infof("Stopped %s", cacheServer.getQualifier());
+        log.infof("--DC: Stopped %s", cacheServer.getQualifier());
     }
 
-
     /**
      * Sets time offset on all the started containers.
      *

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -47,6 +47,7 @@ import org.keycloak.testsuite.pages.ProceedPage;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import org.hamcrest.Matchers;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertThat;
 
@@ -84,6 +85,8 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
       @JmxInfinispanCacheStatistics(dc=DC.FIRST, dcNodeIndex=1, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc0Node1Statistics,
       @JmxInfinispanCacheStatistics(dc=DC.SECOND, dcNodeIndex=0, cacheName=InfinispanConnectionProvider.ACTION_TOKEN_CACHE) InfinispanStatistics cacheDc1Node0Statistics,
       @JmxInfinispanChannelStatistics() InfinispanStatistics channelStatisticsCrossDc) throws Exception {
+        log.debug("--DC: START sendResetPasswordEmailSuccessWorksInCrossDc");
+        
         startBackendNode(DC.FIRST, 1);
         cacheDc0Node1Statistics.waitToBecomeAvailable(10, TimeUnit.SECONDS);
 
@@ -133,7 +136,7 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
                 }
         );
 
-        assertEquals("Your account has been updated.", PageUtils.getPageTitle(driver));
+        assertThat(PageUtils.getPageTitle(driver), containsString("Your account has been updated."));
 
         // Verify that there was an action token added in the node which was targetted by the link
         assertThat(cacheDc0Node0Statistics.getSingleStatistics(Constants.STAT_CACHE_NUMBER_OF_ENTRIES), greaterThan(originalNumberOfEntries));
@@ -148,10 +151,12 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
         );
 
         errorPage.assertCurrent();
+        log.debug("--DC: END sendResetPasswordEmailSuccessWorksInCrossDc");
     }
 
     @Test
     public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
+        log.debug("--DC: START sendResetPasswordEmailAfterNewNodeAdded");
         disableDcOnLoadBalancer(DC.SECOND);
 
         UserRepresentation userRep = new UserRepresentation();
@@ -183,17 +188,15 @@ public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
         assertEquals("Your account has been updated.", PageUtils.getPageTitle(driver));
 
         disableDcOnLoadBalancer(DC.FIRST);
-        getManuallyStartedBackendNodes(DC.SECOND)
+        startBackendNode(DC.SECOND, 1);
-          .findFirst()
+        enableLoadBalancerNode(DC.SECOND, 1);
-          .ifPresent(c -> {
-              containerController.start(c.getQualifier());
-              loadBalancerCtrl.enableBackendNodeByName(c.getQualifier());
-          });
 
         Retry.execute(() -> {
             driver.navigate().to(link);
             errorPage.assertCurrent();
         }, 3, 400);
+
+        log.debug("--DC: END sendResetPasswordEmailAfterNewNodeAdded");
     }
 
 }

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/BruteForceCrossDCTest.java

@@ -64,7 +64,9 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
                 AbstractAdminCrossDCTest.class,
                 AbstractCrossDCTest.class,
                 AbstractTestRealmKeycloakTest.class,
-                KeycloakTestingClient.class
+                KeycloakTestingClient.class,
+                Keycloak.class,
+                RealmResource.class
         );
     }
     
@@ -76,12 +78,15 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
                 AbstractAdminCrossDCTest.class,
                 AbstractCrossDCTest.class,
                 AbstractTestRealmKeycloakTest.class,
-                KeycloakTestingClient.class
+                KeycloakTestingClient.class,
+                Keycloak.class,
+                RealmResource.class
         );
     }
     
     @Before
     public void beforeTest() {
+        log.debug("--DC: creating test realm");
         try {
             adminClient.realm(REALM_NAME).remove();
         } catch (NotFoundException ignore) {
@@ -229,13 +234,8 @@ public class BruteForceCrossDCTest extends AbstractAdminCrossDCTest {
         addUserLoginFailure(getTestingClientForStartedNodeInDc(0));
         assertStatistics("After create entry1", 1, 0, 1);
 
-        AbstractConcurrencyTest.KeycloakRunnable runnable = new AbstractConcurrencyTest.KeycloakRunnable() {
+        AbstractConcurrencyTest.KeycloakRunnable runnable = (int threadIndex, Keycloak keycloak, RealmResource realm1) -> {
-
+            createBruteForceFailures(1, "login-test-1");
-            @Override
-            public void run(int threadIndex, Keycloak keycloak, RealmResource realm) throws Throwable {
-                createBruteForceFailures(1, "login-test-1");
-            }
-
         };
 
         AbstractConcurrencyTest.run(2, 20, this, runnable);

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ConcurrentLoginCrossDCTest.java

@@ -17,7 +17,7 @@
 
 package org.keycloak.testsuite.crossdc;
 
-import org.junit.Assert;
+import java.util.ArrayList;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.admin.client.resource.RealmResource;
 import java.util.List;
@@ -36,7 +36,6 @@ import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.LaxRedirectStrategy;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /**
@@ -57,16 +56,40 @@ public class ConcurrentLoginCrossDCTest extends ConcurrentLoginTest {
 
     @Override
     public void beforeAbstractKeycloakTestRealmImport() {
-        log.debug("Initializing load balancer - enabling all started nodes across DCs");
+        log.debug("--DC: Starting cacheServers if not started already");
+        suiteContext.getCacheServersInfo().stream()
+                .filter((containerInfo) -> !containerInfo.isStarted())
+                .map(ContainerInfo::getQualifier)
+                .forEach(containerController::start);
+        
+        log.debug("--DC: Initializing load balancer - enabling all started nodes across DCs");
         this.loadBalancerCtrl.disableAllBackendNodes();
 
         this.suiteContext.getDcAuthServerBackendsInfo().stream()
                 .flatMap(List::stream)
-                .filter(ContainerInfo::isStarted)
+                .filter((containerInfo) -> !containerInfo.getQualifier().contains("manual"))
+                .filter((containerInfo) -> !containerInfo.isStarted())
                 .map(ContainerInfo::getQualifier)
-                .forEach(loadBalancerCtrl::enableBackendNodeByName);
+                .forEach((nodeName) -> {
+                    containerController.start(nodeName);
+                    loadBalancerCtrl.enableBackendNodeByName(nodeName);
+                });
     }
 
+    @Override
+    public void postAfterAbstractKeycloak() {
+        log.debug("--DC: postAfterAbstractKeycloak");
+        suiteContext.getDcAuthServerBackendsInfo().stream()
+                .flatMap(List::stream)
+                .filter(ContainerInfo::isStarted)
+                .map(ContainerInfo::getQualifier)
+                .forEach(containerController::stop);
+
+        loadBalancerCtrl.disableAllBackendNodes();
+        
+        //realms is already removed and this prevents another removal in AuthServerTestEnricher.afterClass
+        testContext.setTestRealmReps(new ArrayList<>());
+    }
 
     @Test
     public void concurrentLoginWithRandomDcFailures() throws Throwable {

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/SessionsPreloadCrossDCTest.java

@@ -15,82 +15,37 @@
  * limitations under the License.
  */
 
-package org.keycloak.testsuite.crossdc.manual;
+package org.keycloak.testsuite.crossdc;
 
 import java.util.LinkedList;
 import java.util.List;
 
+import org.junit.Before;
 import org.junit.Test;
 import org.keycloak.OAuth2Constants;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.representations.idm.RealmRepresentation;
 import org.keycloak.testsuite.Assert;
 import org.keycloak.testsuite.admin.ApiUtil;
-import org.keycloak.testsuite.arquillian.AuthServerTestEnricher;
-import org.keycloak.testsuite.crossdc.AbstractAdminCrossDCTest;
-import org.keycloak.testsuite.crossdc.DC;
 import org.keycloak.testsuite.util.OAuthClient;
-import org.junit.Assume;
 
 /**
  * Tests userSessions and offline sessions preloading at startup
  *
- * This test requires that lifecycle of infinispan/JDG servers is managed by testsuite, so you need to run with:
- *
- * -Dmanual.mode=true
- *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
 
     private static final int SESSIONS_COUNT = 10;
 
-    @Override
+    @Before
-    public void beforeAbstractKeycloakTest() throws Exception {
+    public void beforeSessionsPreloadCrossDCTest() throws Exception {
-        // Doublecheck we are in manual mode
+        // Start DC1 and only All Keycloak nodes on DC2 are stopped
-        Assume.assumeTrue("The test requires to be executed with manual.mode=true", suiteContext.getCacheServersInfo().get(0).isManual());
+        stopBackendNode(DC.SECOND, 0);
-
+        disableDcOnLoadBalancer(DC.SECOND);
-        stopAllCacheServersAndAuthServers();
-
-        // Start DC1 and only the cache container from DC2. All Keycloak nodes on DC2 are stopped
-        containerController.start(getCacheServer(DC.FIRST).getQualifier());
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
-        startBackendNode(DC.FIRST, 0);
-        enableLoadBalancerNode(DC.FIRST, 0);
-
-        super.beforeAbstractKeycloakTest();
     }
 
 
-    // Override as we are in manual mode
-    @Override
-    public void enableOnlyFirstNodeInFirstDc() {
-    }
-
-
-    // Override as we are in manual mode
-    @Override
-    public void terminateManuallyStartedServers() {
-    }
-
-
-
-
-    @Override
-    public void afterAbstractKeycloakTest() {
-        super.afterAbstractKeycloakTest();
-
-        // Remove realms now. In @AfterClass servers are already shutdown
-        AuthServerTestEnricher.removeTestRealms(testContext, adminClient);
-        testContext.setTestRealmReps(null);
-
-        adminClient.close();
-        adminClient = null;
-        testContext.setAdminClient(null);
-
-        stopAllCacheServersAndAuthServers();
-    }
-
     private void stopAllCacheServersAndAuthServers() {
         log.infof("Going to stop all auth servers");
 
@@ -158,8 +113,8 @@ public class SessionsPreloadCrossDCTest extends AbstractAdminCrossDCTest {
         stopAllCacheServersAndAuthServers();
 
         // Start cache containers on both DC1 and DC2
-        containerController.start(getCacheServer(DC.FIRST).getQualifier());
+        startCacheServer(DC.FIRST);
-        containerController.start(getCacheServer(DC.SECOND).getQualifier());
+        startCacheServer(DC.SECOND);
 
         // Start Keycloak on DC1. Sessions should be preloaded from DB
         startBackendNode(DC.FIRST, 0);

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -79,13 +79,13 @@
         <property name="browser">${browser}</property>
         <property name="firefox_binary">${firefox_binary}</property>
     </extension>
-    
+
     <engine>
         <!-- This allows manual inspection of deployed archives. -->
         <property name="deploymentExportPath">target/deployments</property>
     </engine>
-    
+
-    <container qualifier="auth-server-undertow" mode="suite" default="true">
+    <container qualifier="auth-server-undertow" mode="manual" default="true">
         <configuration>
             <property name="enabled">${auth.server.undertow} &amp;&amp; ! ${auth.server.crossdc}</property>
             <property name="bindAddress">0.0.0.0</property>
@@ -94,8 +94,8 @@
             <property name="remoteMode">${undertow.remote}</property>
         </configuration>
     </container>
-    
+
-    <container qualifier="auth-server-${auth.server}" mode="suite" default="true">
+    <container qualifier="auth-server-${auth.server}" mode="manual" default="true">
         <configuration>
             <property name="enabled">${auth.server.jboss} &amp;&amp; ! ${auth.server.crossdc}</property>
             <property name="adapterImplClass">${auth.server.adapter.impl.class}</property>
@@ -226,7 +226,7 @@
 
     <!-- Cross DC. Node numbering is [centre #].[node #] -->
     <group qualifier="auth-server-jboss-cross-dc">
-        <container qualifier="cache-server-cross-dc-1" mode="suite" >
+        <container qualifier="cache-server-cross-dc-1" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.crossdc} &amp;&amp; ! ${cache.server.lifecycle.skip}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
@@ -252,7 +252,7 @@
             </configuration>
         </container>
 
-        <container qualifier="cache-server-cross-dc-2" mode="suite" >
+        <container qualifier="cache-server-cross-dc-2" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.crossdc} &amp;&amp; ! ${cache.server.lifecycle.skip}</property>
                 <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
@@ -290,7 +290,7 @@
             </configuration>
         </container>
 
-        <container qualifier="auth-server-undertow-cross-dc-0_1" mode="suite" >
+        <container qualifier="auth-server-undertow-cross-dc-0_1" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
@@ -337,7 +337,7 @@
             </configuration>
         </container>
 
-        <container qualifier="auth-server-undertow-cross-dc-1_1" mode="suite" >
+        <container qualifier="auth-server-undertow-cross-dc-1_1" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.undertow.crossdc}</property>
                 <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
@@ -383,7 +383,7 @@
                 }</property>
             </configuration>
         </container>
-        <container qualifier="auth-server-jboss-cross-dc-0_1" mode="suite" >
+        <container qualifier="auth-server-jboss-cross-dc-0_1" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.jboss.crossdc}</property>
                 <property name="adapterImplClass">${auth.server.adapter.impl.class}</property>
@@ -436,7 +436,7 @@
             </configuration>
         </container>
 
-        <container qualifier="auth-server-jboss-cross-dc-1_1" mode="suite" >
+        <container qualifier="auth-server-jboss-cross-dc-1_1" mode="manual" >
             <configuration>
                 <property name="enabled">${auth.server.jboss.crossdc}</property>
                 <property name="adapterImplClass">${auth.server.adapter.impl.class}</property>

travis-run-tests.sh

@@ -74,24 +74,22 @@ if [ $1 == "server-group4" ]; then
     run-server-tests org.keycloak.testsuite.k*.**.*Test,org.keycloak.testsuite.m*.**.*Test,org.keycloak.testsuite.o*.**.*Test,org.keycloak.testsuite.s*.**.*Test
 fi
 
-if [ $1 == "crossdc" ]; then
+if [ $1 == "crossdc1" ]; then
+    cd testsuite/integration-arquillian
+    mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan -DskipTests
+
+    cd tests/base
+    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=org.keycloak.testsuite.crossdc.**.* 2>&1 |
+        java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
+    exit ${PIPESTATUS[0]}
+fi
+
+if [ $1 == "crossdc2" ]; then
     cd testsuite/integration-arquillian
     mvn install -B -nsu -Pauth-servers-crossdc-jboss,auth-server-wildfly,cache-server-infinispan,app-server-wildfly -DskipTests
 
     cd tests/base
-    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,app-server-wildfly -Dtest=*.crossdc.**.* 2>&1 |
+    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,app-server-wildfly -Dtest=org.keycloak.testsuite.adapter.**.crossdc.**.* 2>&1 |
         java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
-    BASE_TESTS_STATUS=${PIPESTATUS[0]}
+    exit ${PIPESTATUS[0]}
-
+fi
-    mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true 2>&1 |
-        java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer
-    MANUAL_TESTS_STATUS=${PIPESTATUS[0]}
-
-    echo "BASE_TESTS_STATUS=$BASE_TESTS_STATUS, MANUAL_TESTS_STATUS=$MANUAL_TESTS_STATUS";
-    if [ $BASE_TESTS_STATUS -eq 0 -a $MANUAL_TESTS_STATUS -eq 0 ]; then
-        exit 0;
-    else
-        exit 1;
-    fi;
-
-fi