KEYCLOAK-4189 - Cross DC testing

common/src/main/java/org/keycloak/common/util/SystemEnvProperties.java

@@ -17,6 +17,8 @@
 
 package org.keycloak.common.util;
 
+import java.util.Collections;
+import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -24,9 +26,21 @@ import java.util.Properties;
  */
 public class SystemEnvProperties extends Properties {
 
+    private final Map<String, String> overrides;
+
+    public SystemEnvProperties(Map<String, String> overrides) {
+        this.overrides = overrides;
+    }
+
+    public SystemEnvProperties() {
+        this.overrides = Collections.EMPTY_MAP;
+    }
+
     @Override
     public String getProperty(String key) {
-        if (key.startsWith("env.")) {
+        if (overrides.containsKey(key)) {
+            return overrides.get(key);
+        } else if (key.startsWith("env.")) {
             return System.getenv().get(key.substring(4));
         } else {
             return System.getProperty(key);

misc/CrossDataCenter.md

@@ -3,6 +3,8 @@ Test Cross-Data-Center scenario (test with external JDG server)
 
 These are temporary notes. This docs should be removed once we have cross-DC support finished and properly documented. 
 
+Note that these steps are already automated, see Cross-DC tests section in [HOW-TO-RUN.md](../testsuite/integration-arquillian/HOW-TO-RUN.md) document.
+ 
 What is working right now is:
 - Propagating of invalidation messages for "realms" and "users" caches
 - All the other things provided by ClusterProvider, which is:

model/infinispan/src/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java

@@ -154,7 +154,8 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
 
         if (clustered) {
             String nodeName = config.get("nodeName", System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME));
-            configureTransport(gcb, nodeName);
+            String jgroupsUdpMcastAddr = config.get("jgroupsUdpMcastAddr", System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR));
+            configureTransport(gcb, nodeName, jgroupsUdpMcastAddr);
         }
         gcb.globalJmxStatistics().allowDuplicateDomains(allowDuplicateJMXDomains);
 
@@ -317,24 +318,40 @@ public class DefaultInfinispanConnectionProviderFactory implements InfinispanCon
         return cb.build();
     }
 
-    protected void configureTransport(GlobalConfigurationBuilder gcb, String nodeName) {
+    private static final Object CHANNEL_INIT_SYNCHRONIZER = new Object();
+
+    protected void configureTransport(GlobalConfigurationBuilder gcb, String nodeName, String jgroupsUdpMcastAddr) {
         if (nodeName == null) {
             gcb.transport().defaultTransport();
         } else {
             FileLookup fileLookup = FileLookupFactory.newInstance();
 
-            try {
+            synchronized (CHANNEL_INIT_SYNCHRONIZER) {
-                // Compatibility with Wildfly
+                String originalMcastAddr = System.getProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
-                JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()));
+                if (jgroupsUdpMcastAddr == null) {
-                channel.setName(nodeName);
+                    System.getProperties().remove(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
-                JGroupsTransport transport = new JGroupsTransport(channel);
+                } else {
+                    System.setProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR, jgroupsUdpMcastAddr);
+                }
+                try {
+                    // Compatibility with Wildfly
+                    JChannel channel = new JChannel(fileLookup.lookupFileLocation("default-configs/default-jgroups-udp.xml", this.getClass().getClassLoader()));
+                    channel.setName(nodeName);
+                    JGroupsTransport transport = new JGroupsTransport(channel);
 
-                gcb.transport().nodeName(nodeName);
+                    gcb.transport().nodeName(nodeName);
-                gcb.transport().transport(transport);
+                    gcb.transport().transport(transport);
 
-                logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
+                    logger.infof("Configured jgroups transport with the channel name: %s", nodeName);
-            } catch (Exception e) {
+                } catch (Exception e) {
-                throw new RuntimeException(e);
+                    throw new RuntimeException(e);
+                } finally {
+                    if (originalMcastAddr == null) {
+                        System.getProperties().remove(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR);
+                    } else {
+                        System.setProperty(InfinispanConnectionProvider.JGROUPS_UDP_MCAST_ADDR, originalMcastAddr);
+                    }
+                }
             }
         }
     }

model/infinispan/src/main/java/org/keycloak/connections/infinispan/InfinispanConnectionProvider.java

@@ -53,6 +53,7 @@ public interface InfinispanConnectionProvider extends Provider {
 
     // System property used on Wildfly to identify distributedCache address and sticky session route
     String JBOSS_NODE_NAME = "jboss.node.name";
+    String JGROUPS_UDP_MCAST_ADDR = "jgroups.udp.mcast_addr";
 
 
     <K, V> Cache<K, V> getCache(String name);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/CacheManager.java

@@ -220,7 +220,7 @@ public abstract class CacheManager {
 
         addInvalidationsFromEvent(event, invalidations);
 
-        getLogger().debugf("Invalidating %d cache items after received event %s", invalidations.size(), event);
+        getLogger().debugf("[%s] Invalidating %d cache items after received event %s", cache.getCacheManager().getAddress(), invalidations.size(), event);
 
         for (String invalidation : invalidations) {
             invalidateObject(invalidation);

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/AddInvalidatedActionTokenEvent.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.keycloak.models.cache.infinispan;
+package org.keycloak.models.cache.infinispan.events;
 
 import org.keycloak.cluster.ClusterEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;

model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/events/RemoveActionTokensSpecificEvent.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.keycloak.models.cache.infinispan;
+package org.keycloak.models.cache.infinispan.events;
 
 import org.keycloak.cluster.ClusterEvent;
 

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProvider.java

@@ -19,8 +19,8 @@ package org.keycloak.models.sessions.infinispan;
 import org.keycloak.cluster.ClusterProvider;
 import org.keycloak.models.*;
 
-import org.keycloak.models.cache.infinispan.AddInvalidatedActionTokenEvent;
+import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
-import org.keycloak.models.cache.infinispan.RemoveActionTokensSpecificEvent;
+import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
 import java.util.*;
@@ -58,7 +58,12 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
         ActionTokenValueEntity tokenValue = new ActionTokenValueEntity(notes);
 
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        this.tx.notify(cluster, InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS, new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue), false);
+        AddInvalidatedActionTokenEvent event = new AddInvalidatedActionTokenEvent(tokenKey, key.getExpiration(), tokenValue);
+        this.tx.notify(cluster, generateActionTokenEventId(), event, false);
+    }
+
+    private static String generateActionTokenEventId() {
+        return InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS + "/" + UUID.randomUUID();
     }
 
     @Override
@@ -93,6 +98,6 @@ public class InfinispanActionTokenStoreProvider implements ActionTokenStoreProvi
         }
 
         ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-        this.tx.notify(cluster, InfinispanActionTokenStoreProviderFactory.ACTION_TOKEN_EVENTS, new RemoveActionTokensSpecificEvent(userId, actionId), false);
+        this.tx.notify(cluster, generateActionTokenEventId(), new RemoveActionTokensSpecificEvent(userId, actionId), false);
     }
 }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanActionTokenStoreProviderFactory.java

@@ -23,14 +23,16 @@ import org.keycloak.common.util.Time;
 import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.models.*;
 
-import org.keycloak.models.cache.infinispan.AddInvalidatedActionTokenEvent;
+import org.keycloak.models.cache.infinispan.events.AddInvalidatedActionTokenEvent;
-import org.keycloak.models.cache.infinispan.RemoveActionTokensSpecificEvent;
+import org.keycloak.models.cache.infinispan.events.RemoveActionTokensSpecificEvent;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenValueEntity;
 import org.keycloak.models.sessions.infinispan.entities.ActionTokenReducedKey;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import org.infinispan.Cache;
 import org.infinispan.context.Flag;
+import org.infinispan.remoting.transport.Address;
+import org.jboss.logging.Logger;
 
 /**
  *
@@ -38,6 +40,10 @@ import org.infinispan.context.Flag;
  */
 public class InfinispanActionTokenStoreProviderFactory implements ActionTokenStoreProviderFactory {
 
+    private static final Logger LOG = Logger.getLogger(InfinispanActionTokenStoreProviderFactory.class);
+
+    private volatile Cache<ActionTokenReducedKey, ActionTokenValueEntity> actionTokenCache;
+
     public static final String ACTION_TOKEN_EVENTS = "ACTION_TOKEN_EVENTS";
 
     /**
@@ -49,34 +55,7 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto
 
     @Override
     public ActionTokenStoreProvider create(KeycloakSession session) {
-        InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
+        return new InfinispanActionTokenStoreProvider(session, this.actionTokenCache);
-        Cache<ActionTokenReducedKey, ActionTokenValueEntity> actionTokenCache = connections.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE);
-
-        ClusterProvider cluster = session.getProvider(ClusterProvider.class);
-
-        cluster.registerListener(ACTION_TOKEN_EVENTS, event -> {
-            if (event instanceof RemoveActionTokensSpecificEvent) {
-                RemoveActionTokensSpecificEvent e = (RemoveActionTokensSpecificEvent) event;
-
-                actionTokenCache
-                  .getAdvancedCache()
-                  .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD)
-                  .keySet()
-                  .stream()
-                  .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId()))
-                  .forEach(actionTokenCache::remove);
-            } else if (event instanceof AddInvalidatedActionTokenEvent) {
-                AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event;
-
-                if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) {
-                    actionTokenCache.put(e.getKey(), e.getTokenValue());
-                } else {
-                    actionTokenCache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS);
-                }
-            }
-        });
-
-        return new InfinispanActionTokenStoreProvider(session, actionTokenCache);
     }
 
     @Override
@@ -84,8 +63,57 @@ public class InfinispanActionTokenStoreProviderFactory implements ActionTokenSto
         this.config = config;
     }
 
+    private static Cache<ActionTokenReducedKey, ActionTokenValueEntity> initActionTokenCache(KeycloakSession session) {
+        InfinispanConnectionProvider connections = session.getProvider(InfinispanConnectionProvider.class);
+        Cache<ActionTokenReducedKey, ActionTokenValueEntity> cache = connections.getCache(InfinispanConnectionProvider.ACTION_TOKEN_CACHE);
+        final Address cacheAddress = cache.getCacheManager().getAddress();
+
+        ClusterProvider cluster = session.getProvider(ClusterProvider.class);
+
+        cluster.registerListener(ClusterProvider.ALL, event -> {
+            if (event instanceof RemoveActionTokensSpecificEvent) {
+                RemoveActionTokensSpecificEvent e = (RemoveActionTokensSpecificEvent) event;
+
+                LOG.debugf("[%s] Removing token invalidation for user+action: userId=%s, actionId=%s", cacheAddress, e.getUserId(), e.getActionId());
+
+                cache
+                  .getAdvancedCache()
+                  .withFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_CACHE_LOAD)
+                  .keySet()
+                  .stream()
+                  .filter(k -> Objects.equals(k.getUserId(), e.getUserId()) && Objects.equals(k.getActionId(), e.getActionId()))
+                  .forEach(cache::remove);
+            } else if (event instanceof AddInvalidatedActionTokenEvent) {
+                AddInvalidatedActionTokenEvent e = (AddInvalidatedActionTokenEvent) event;
+
+                LOG.debugf("[%s] Invalidating token %s", cacheAddress, e.getKey());
+                if (e.getExpirationInSecs() == DEFAULT_CACHE_EXPIRATION) {
+                    cache.put(e.getKey(), e.getTokenValue());
+                } else {
+                    cache.put(e.getKey(), e.getTokenValue(), e.getExpirationInSecs() - Time.currentTime(), TimeUnit.SECONDS);
+                }
+            }
+        });
+
+        LOG.debugf("[%s] Registered cluster listeners", cacheAddress);
+
+        return cache;
+    }
+
     @Override
     public void postInit(KeycloakSessionFactory factory) {
+        Cache<ActionTokenReducedKey, ActionTokenValueEntity> cache = this.actionTokenCache;
+
+        // It is necessary to put the cache initialization here, otherwise the cache would be initialized lazily, that
+        // means also listeners will start only after first cache initialization - that would be too late
+        if (cache == null) {
+            synchronized (this) {
+                cache = this.actionTokenCache;
+                if (cache == null) {
+                    this.actionTokenCache = initActionTokenCache(factory.create());
+                }
+            }
+        }
     }
 
     @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanAuthenticationSessionProviderFactory.java

@@ -92,7 +92,7 @@ public class InfinispanAuthenticationSessionProviderFactory implements Authentic
                     ClusterProvider cluster = session.getProvider(ClusterProvider.class);
                     cluster.registerListener(AUTHENTICATION_SESSION_EVENTS, this::updateAuthNotes);
 
-                    log.debug("Registered cluster listeners");
+                    log.debugf("[%s] Registered cluster listeners", authSessionsCache.getCacheManager().getAddress());
                 }
             }
         }

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/ActionTokenReducedKey.java

@@ -81,6 +81,11 @@ public class ActionTokenReducedKey implements Serializable {
           && Objects.equals(this.actionVerificationNonce, other.getActionVerificationNonce());
     }
 
+    @Override
+    public String toString() {
+        return "userId=" + userId + ", actionId=" + actionId + ", actionVerificationNonce=" + actionVerificationNonce;
+    }
+
     public static class ExternalizerImpl implements Externalizer<ActionTokenReducedKey> {
 
         @Override

model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/entities/ActionTokenValueEntity.java

@@ -53,7 +53,7 @@ public class ActionTokenValueEntity implements ActionTokenValueModel {
         public void writeObject(ObjectOutput output, ActionTokenValueEntity t) throws IOException {
             output.writeByte(VERSION_1);
 
-            output.writeBoolean(! t.notes.isEmpty());
+            output.writeBoolean(t.notes.isEmpty());
             if (! t.notes.isEmpty()) {
                 output.writeObject(t.notes);
             }

services/src/main/java/org/keycloak/services/resources/KeycloakApplication.java

@@ -70,8 +70,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URL;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.StringTokenizer;
@@ -88,6 +90,8 @@ public class KeycloakApplication extends Application {
 
     public static final String KEYCLOAK_EMBEDDED = "keycloak.embedded";
 
+    public static final String SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES = "keycloak.server.context.config.property-overrides";
+
     private static final Logger logger = Logger.getLogger(KeycloakApplication.class);
 
     protected boolean embedded = false;
@@ -287,7 +291,13 @@ public class KeycloakApplication extends Application {
             }
 
             if (node != null) {
-                Properties properties = new SystemEnvProperties();
+                Map<String, String> propertyOverridesMap = new HashMap<>();
+                String propertyOverrides = context.getInitParameter(SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES);
+                if (context.getInitParameter(SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES) != null) {
+                    JsonNode jsonObj = new ObjectMapper().readTree(propertyOverrides);
+                    jsonObj.fields().forEachRemaining(e -> propertyOverridesMap.put(e.getKey(), e.getValue().asText()));
+                }
+                Properties properties = new SystemEnvProperties(propertyOverridesMap);
                 Config.init(new JsonConfigProvider(node, properties));
             } else {
                 throw new RuntimeException("Keycloak config not found.");

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -417,4 +417,22 @@ and argument: `-p 8181`
 
 3) Run loadbalancer (class `SimpleUndertowLoadBalancer`) without arguments and system properties. Loadbalancer runs on port 8180, so you can access Keycloak on `http://localhost:8180/auth`     
 
+## Cross-DC tests
 
+Cross-DC tests use 2 data centers, each with one automatically started and one manually controlled backend servers
+(currently only Keycloak on Undertow), and 1 frontend loadbalancer server node that sits in front of all servers.
+The browser usually communicates directly with the frontent node and the test controls where the HTTP requests
+land by adjusting load balancer configuration (e.g. to direct the traffic to only a single DC).
+
+For an example of a test, see [org.keycloak.testsuite.crossdc.ActionTokenCrossDCTest](tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java).
+
+The cross DC requires setting a profile specifying used cache server (currently only Infinispan) by specifying
+`cache-server-infinispan` profile in maven.
+
+#### Run Cross-DC Tests from Maven
+
+Run the following command (adjust the test specification according to your needs):
+
+  `mvn -Pcache-server-infinispan -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base test`
+
+_Someone using IntelliJ IDEA, please describe steps for that IDE_

testsuite/integration-arquillian/pom.xml

@@ -46,6 +46,7 @@
         <arquillian-drone.version>2.0.1.Final</arquillian-drone.version>
         <arquillian-graphene.version>2.1.0.Alpha3</arquillian-graphene.version>
         <arquillian-wildfly-container.version>2.1.0.Alpha2</arquillian-wildfly-container.version>
+        <arquillian-infinispan-container.version>1.2.0.Beta2</arquillian-infinispan-container.version>
         <version.shrinkwrap.resolvers>2.2.2</version.shrinkwrap.resolvers>
         <undertow-embedded.version>1.0.0.Alpha2</undertow-embedded.version>
 
@@ -87,6 +88,11 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>org.infinispan.arquillian.container</groupId>
+                <artifactId>infinispan-arquillian-impl</artifactId>
+                <version>${arquillian-infinispan-container.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.wildfly.arquillian</groupId>
                 <artifactId>wildfly-arquillian-container-managed</artifactId>

testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/arquillian/LoadBalancerController.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public interface LoadBalancerController {
+
+    void enableAllBackendNodes();
+
+    void disableAllBackendNodes();
+
+    void enableBackendNodeByName(String nodeName);
+
+    void disableBackendNodeByName(String nodeName);
+
+}

testsuite/integration-arquillian/servers/auth-server/undertow/pom.xml

@@ -70,6 +70,11 @@
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-undertow-adapter</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.keycloak.testsuite</groupId>
+            <artifactId>integration-arquillian-testsuite-providers</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.keycloak</groupId>
             <artifactId>keycloak-servlet-filter-adapter</artifactId>

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertow.java

@@ -48,6 +48,8 @@ import org.keycloak.services.filters.KeycloakSessionServletFilter;
 import org.keycloak.services.managers.ApplianceBootstrap;
 import org.keycloak.services.resources.KeycloakApplication;
 
+import org.keycloak.util.JsonSerialization;
+import java.io.IOException;
 import javax.servlet.DispatcherType;
 import javax.servlet.ServletException;
 
@@ -55,6 +57,7 @@ import java.lang.reflect.Field;
 import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
 
 public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUndertowConfiguration> {
 
@@ -75,6 +78,14 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
         di.setContextPath("/auth");
         di.setDeploymentName("Keycloak");
         di.addInitParameter(KeycloakApplication.KEYCLOAK_EMBEDDED, "true");
+        if (configuration.getKeycloakConfigPropertyOverridesMap() != null) {
+            try {
+                di.addInitParameter(KeycloakApplication.SERVER_CONTEXT_CONFIG_PROPERTY_OVERRIDES,
+                  JsonSerialization.writeValueAsString(configuration.getKeycloakConfigPropertyOverridesMap()));
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
 
         di.setDefaultServletConfig(new DefaultServletConfig(true));
         di.addWelcomePage("theme/keycloak/welcome/resources/index.html");
@@ -176,19 +187,14 @@ public class KeycloakOnUndertow implements DeployableContainer<KeycloakOnUnderto
             log.info("Using route: " + configuration.getRoute());
         }
 
-        SetSystemProperty setRouteProperty = new SetSystemProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME, configuration.getRoute());
+        DeploymentInfo di = createAuthServerDeploymentInfo();
-        try {
+        undertow.deploy(di);
-            DeploymentInfo di = createAuthServerDeploymentInfo();
+        ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
-            undertow.deploy(di);
+        sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
-            ResteasyDeployment deployment = (ResteasyDeployment) di.getServletContextAttributes().get(ResteasyDeployment.class.getName());
-            sessionFactory = ((KeycloakApplication) deployment.getApplication()).getSessionFactory();
 
-            setupDevConfig();
+        setupDevConfig();
 
-            log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
+        log.info("Auth server started in " + (System.currentTimeMillis() - start) + " ms\n");
-        } finally {
-            setRouteProperty.revert();
-        }
     }
 
 

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/KeycloakOnUndertowConfiguration.java

@@ -17,6 +17,11 @@
 
 package org.keycloak.testsuite.arquillian.undertow;
 
+import org.keycloak.util.JsonSerialization;
+import com.fasterxml.jackson.core.type.TypeReference;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import org.arquillian.undertow.UndertowContainerConfiguration;
 import org.jboss.arquillian.container.spi.ConfigurationException;
 import org.jboss.logging.Logger;
@@ -29,6 +34,8 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
     private String resourcesHome;
     private boolean remoteMode;
     private String route;
+    private String keycloakConfigPropertyOverrides;
+    private Map<String, String> keycloakConfigPropertyOverridesMap;
 
     private int bindHttpPortOffset = 0;
 
@@ -72,6 +79,18 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
         this.remoteMode = remoteMode;
     }
 
+    public String getKeycloakConfigPropertyOverrides() {
+        return keycloakConfigPropertyOverrides;
+    }
+
+    public void setKeycloakConfigPropertyOverrides(String keycloakConfigPropertyOverrides) {
+        this.keycloakConfigPropertyOverrides = keycloakConfigPropertyOverrides;
+    }
+
+    public Map<String, String> getKeycloakConfigPropertyOverridesMap() {
+        return keycloakConfigPropertyOverridesMap;
+    }
+
     @Override
     public void validate() throws ConfigurationException {
         super.validate();
@@ -81,6 +100,15 @@ public class KeycloakOnUndertowConfiguration extends UndertowContainerConfigurat
         setBindHttpPort(newPort);
         log.info("KeycloakOnUndertow will listen on port: " + newPort);
         
+        if (this.keycloakConfigPropertyOverrides != null) {
+            try {
+                TypeReference<HashMap<String,Object>> typeRef = new TypeReference<HashMap<String,Object>>() {};
+                this.keycloakConfigPropertyOverridesMap = JsonSerialization.sysPropertiesAwareMapper.readValue(this.keycloakConfigPropertyOverrides, typeRef);
+            } catch (IOException ex) {
+                throw new ConfigurationException(ex);
+            }
+        }
+
         // TODO validate workerThreads
         
     }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/SetSystemProperty.java

@@ -17,13 +17,15 @@
 
 package org.keycloak.testsuite.arquillian.undertow;
 
+import java.io.Closeable;
+
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-class SetSystemProperty {
+class SetSystemProperty implements Closeable {
 
-    private String name;
+    private final String name;
-    private String oldValue;
+    private final String oldValue;
 
     public SetSystemProperty(String name, String value) {
         this.name = name;
@@ -50,4 +52,9 @@ class SetSystemProperty {
         }
     }
 
+    @Override
+    public void close() {
+        revert();
+    }
+
 }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancer.java

@@ -18,7 +18,6 @@
 package org.keycloak.testsuite.arquillian.undertow.lb;
 
 import java.net.URI;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -36,6 +35,8 @@ import io.undertow.util.AttachmentKey;
 import io.undertow.util.Headers;
 import org.jboss.logging.Logger;
 import org.keycloak.services.managers.AuthenticationSessionManager;
+import java.util.LinkedHashMap;
+import java.util.StringTokenizer;
 
 /**
  * Loadbalancer on embedded undertow. Supports sticky session over "AUTH_SESSION_ID" cookie and failover to different node when sticky node not available.
@@ -53,8 +54,9 @@ public class SimpleUndertowLoadBalancer {
 
     private final String host;
     private final int port;
-    private final String nodesString;
+    private final Map<String, URI> backendNodes;
     private Undertow undertow;
+    private LoadBalancingProxyClient lb;
 
 
     public static void main(String[] args) throws Exception {
@@ -77,15 +79,14 @@ public class SimpleUndertowLoadBalancer {
     public SimpleUndertowLoadBalancer(String host, int port, String nodesString) {
         this.host = host;
         this.port = port;
-        this.nodesString = nodesString;
+        this.backendNodes = parseNodes(nodesString);
-        log.infof("Keycloak nodes: %s", nodesString);
+        log.infof("Keycloak nodes: %s", backendNodes);
     }
 
 
     public void start() {
-        Map<String, String> nodes = parseNodes(nodesString);
         try {
-            HttpHandler proxyHandler = createHandler(nodes);
+            HttpHandler proxyHandler = createHandler();
 
             undertow = Undertow.builder()
                     .addHttpListener(port, host)
@@ -104,24 +105,51 @@ public class SimpleUndertowLoadBalancer {
         undertow.stop();
     }
 
+    public void enableAllBackendNodes() {
+        backendNodes.forEach((route, uri) -> {
+            lb.removeHost(uri);
+            lb.addHost(uri, route);
+        });
+    }
 
-    static Map<String, String> parseNodes(String nodes) {
+    public void disableAllBackendNodes() {
-        String[] nodesArray = nodes.split(",");
+        log.debugf("Load balancer: disabling all nodes");
-        Map<String, String> result = new HashMap<>();
+        backendNodes.values().forEach(lb::removeHost);
+    }
 
-        for (String nodeStr : nodesArray) {
+    public void enableBackendNodeByName(String nodeName) {
-            String[] node = nodeStr.trim().split("=");
+        URI uri = backendNodes.get(nodeName);
-            if (node.length != 2) {
+        if (uri == null) {
-                throw new IllegalArgumentException("Illegal node format in the configuration: " + nodeStr);
+            throw new IllegalArgumentException("Invalid node: " + nodeName);
-            }
+        }
-            result.put(node[0].trim(), node[1].trim());
+        log.debugf("Load balancer: enabling node %s", nodeName);
+        lb.addHost(uri, nodeName);
+    }
+
+    public void disableBackendNodeByName(String nodeName) {
+        URI uri = backendNodes.get(nodeName);
+        if (uri == null) {
+            throw new IllegalArgumentException("Invalid node: " + nodeName);
+        }
+        log.debugf("Load balancer: disabling node %s", nodeName);
+        lb.removeHost(uri);
+    }
+
+    static Map<String, URI> parseNodes(String nodes) {
+        StringTokenizer st = new StringTokenizer(nodes, ",");
+        Map<String, URI> result = new LinkedHashMap<>();
+
+        while (st.hasMoreElements()) {
+            String nodeStr = st.nextToken();
+            String[] node = nodeStr.trim().split("=", 2);
+            result.put(node[0].trim(), URI.create(node[1].trim()));
         }
 
         return result;
     }
 
 
-    private HttpHandler createHandler(Map<String, String> backendNodes) throws Exception {
+    private HttpHandler createHandler() throws Exception {
 
         // TODO: configurable options if needed
         String sessionCookieNames = AuthenticationSessionManager.AUTH_SESSION_ID;
@@ -133,15 +161,7 @@ public class SimpleUndertowLoadBalancer {
         int connectionIdleTimeout = 60;
         int maxRetryAttempts = backendNodes.size() - 1;
 
-        final LoadBalancingProxyClient lb = new CustomLoadBalancingClient(new ExclusivityChecker() {
+        lb = new CustomLoadBalancingClient(exchange -> exchange.getRequestHeaders().contains(Headers.UPGRADE), maxRetryAttempts)
-
-            @Override
-            public boolean isExclusivityRequired(HttpServerExchange exchange) {
-                //we always create a new connection for upgrade requests
-                return exchange.getRequestHeaders().contains(Headers.UPGRADE);
-            }
-
-        }, maxRetryAttempts)
                 .setConnectionsPerThread(connectionsPerThread)
                 .setMaxQueueSize(requestQueueSize)
                 .setSoftMaxConnectionsPerThread(cachedConnectionsPerThread)
@@ -152,13 +172,10 @@ public class SimpleUndertowLoadBalancer {
             lb.addSessionCookieName(id);
         }
 
-        for (Map.Entry<String, String> node : backendNodes.entrySet()) {
+        backendNodes.forEach((route, uri) -> {
-            String route = node.getKey();
-            URI uri = new URI(node.getValue());
-
             lb.addHost(uri, route);
-            log.infof("Added host: %s, route: %s", uri.toString(), route);
+            log.debugf("Added host: %s, route: %s", uri.toString(), route);
-        }
+        });
 
         ProxyHandler handler = new ProxyHandler(lb, maxTime, ResponseCodeHandler.HANDLE_404);
         return handler;

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerConfiguration.java

@@ -19,13 +19,17 @@ package org.keycloak.testsuite.arquillian.undertow.lb;
 
 import org.arquillian.undertow.UndertowContainerConfiguration;
 import org.jboss.arquillian.container.spi.ConfigurationException;
+import org.jboss.logging.Logger;
 
 /**
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
 public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerConfiguration {
 
+    protected static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancerConfiguration.class);
+
     private String nodes = SimpleUndertowLoadBalancer.DEFAULT_NODES;
+    private int bindHttpPortOffset = 0;
 
     public String getNodes() {
         return nodes;
@@ -35,6 +39,14 @@ public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerCo
         this.nodes = nodes;
     }
 
+    public int getBindHttpPortOffset() {
+        return bindHttpPortOffset;
+    }
+
+    public void setBindHttpPortOffset(int bindHttpPortOffset) {
+        this.bindHttpPortOffset = bindHttpPortOffset;
+    }
+
     @Override
     public void validate() throws ConfigurationException {
         super.validate();
@@ -44,5 +56,11 @@ public class SimpleUndertowLoadBalancerConfiguration extends UndertowContainerCo
         } catch (Exception e) {
             throw new ConfigurationException(e);
         }
+
+        int basePort = getBindHttpPort();
+        int newPort = basePort + bindHttpPortOffset;
+        setBindHttpPort(newPort);
+        log.info("SimpleUndertowLoadBalancer will listen on port: " + newPort);
+
     }
 }

testsuite/integration-arquillian/servers/auth-server/undertow/src/main/java/org/keycloak/testsuite/arquillian/undertow/lb/SimpleUndertowLoadBalancerContainer.java

@@ -25,13 +25,14 @@ import org.jboss.arquillian.container.spi.client.protocol.metadata.ProtocolMetaD
 import org.jboss.logging.Logger;
 import org.jboss.shrinkwrap.api.Archive;
 import org.jboss.shrinkwrap.descriptor.api.Descriptor;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
 
 /**
  * Arquillian container over {@link SimpleUndertowLoadBalancer}
  *
  * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
  */
-public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<SimpleUndertowLoadBalancerConfiguration> {
+public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<SimpleUndertowLoadBalancerConfiguration>, LoadBalancerController {
 
     private static final Logger log = Logger.getLogger(SimpleUndertowLoadBalancerContainer.class);
 
@@ -84,4 +85,24 @@ public class SimpleUndertowLoadBalancerContainer implements DeployableContainer<
     public void undeploy(Descriptor descriptor) throws DeploymentException {
         throw new UnsupportedOperationException("Not implemented");
     }
+
+    @Override
+    public void enableAllBackendNodes() {
+        this.container.enableAllBackendNodes();
+    }
+
+    @Override
+    public void disableAllBackendNodes() {
+        this.container.disableAllBackendNodes();
+    }
+
+    @Override
+    public void enableBackendNodeByName(String nodeName) {
+        this.container.enableBackendNodeByName(nodeName);
+    }
+
+    @Override
+    public void disableBackendNodeByName(String nodeName) {
+        this.container.disableBackendNodeByName(nodeName);
+    }
 }

testsuite/integration-arquillian/servers/cache-server/jboss/assembly.xml

@@ -0,0 +1,46 @@
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly>
+    
+    <id>${cache.server.jboss}</id>
+    
+    <formats>
+        <format>zip</format>
+    </formats>
+
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>${cache.server.jboss.home}</directory>
+            <outputDirectory>cache-server-${cache.server}</outputDirectory>
+            <excludes>
+                <exclude>**/*.sh</exclude>
+            </excludes>
+        </fileSet>
+        <fileSet>
+            <directory>${cache.server.jboss.home}</directory>
+            <outputDirectory>cache-server-${cache.server}</outputDirectory>
+            <includes>
+                <include>**/*.sh</include>
+            </includes>
+            <fileMode>0755</fileMode>
+        </fileSet>
+    </fileSets>
+
+</assembly>

testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-remote-store.xsl

@@ -0,0 +1,42 @@
+<!--
+  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                version="2.0"
+                exclude-result-prefixes="xalan">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:variable name="nsCacheServer" select="'urn:infinispan:server:core:'"/>
+
+    <xsl:template match="//*[local-name()='subsystem' and starts-with(namespace-uri(), $nsCacheServer)]
+                        /*[local-name()='cache-container' and starts-with(namespace-uri(), $nsCacheServer) and @name='local']">
+        <xsl:copy>
+            <xsl:apply-templates select="@* | node()" />
+            <local-cache name="work" start="EAGER" batching="false" />
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>

testsuite/integration-arquillian/servers/cache-server/jboss/common/io.xsl

@@ -0,0 +1,40 @@
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xalan="http://xml.apache.org/xalan"
+                version="2.0"
+                exclude-result-prefixes="xalan">
+
+    <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" xalan:indent-amount="4" standalone="no"/>
+    <xsl:strip-space elements="*"/>
+
+    <xsl:param name="worker.io-threads" select="'16'"/>
+    <xsl:param name="worker.task-max-threads" select="'128'"/>
+    
+    <!--set worker threads-->
+    <xsl:template match="//*[local-name()='worker' and @name='default']">
+        <worker name="default" io-threads="{$worker.io-threads}" task-max-threads="{$worker.task-max-threads}" />
+    </xsl:template>
+
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()" />
+        </xsl:copy>
+    </xsl:template>
+
+</xsl:stylesheet>

testsuite/integration-arquillian/servers/cache-server/jboss/infinispan/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers-cache-server-jboss</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server-infinispan</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server - JBoss - Infinispan</name>
+    
+    <properties>
+        <cache.server>infinispan</cache.server>
+        <cache.server.container>cache-server-${cache.server}</cache.server.container>
+        <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
+        
+        <cache.server.jboss.groupId>org.infinispan.server</cache.server.jboss.groupId>
+        <cache.server.jboss.artifactId>infinispan-server</cache.server.jboss.artifactId>
+        <cache.server.jboss.version>${infinispan.version}</cache.server.jboss.version>
+        <cache.server.jboss.unpacked.folder.name>${cache.server.jboss.artifactId}-${infinispan.version}</cache.server.jboss.unpacked.folder.name>
+        
+        <cache.server.worker.io-threads>${cache.default.worker.io-threads}</cache.server.worker.io-threads>
+        <cache.server.worker.task-max-threads>${cache.default.worker.task-max-threads}</cache.server.worker.task-max-threads>
+    </properties>
+
+</project>

testsuite/integration-arquillian/servers/cache-server/jboss/pom.xml

@@ -0,0 +1,210 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers-cache-server</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server-jboss</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server - JBoss Family</name>
+
+    <properties>
+        <common.resources>${project.parent.basedir}/common</common.resources>
+        <assembly.xml>${project.parent.basedir}/assembly.xml</assembly.xml>
+        <cache.server.jboss.home>${containers.home}/${cache.server.jboss.unpacked.folder.name}</cache.server.jboss.home>
+        <security.xslt>security.xsl</security.xslt>
+    </properties>
+
+    <profiles>
+        
+        <profile>
+            <id>cache-server-jboss-submodules</id>
+            <activation>
+                <file>
+                    <exists>src</exists>
+                </file>
+            </activation>
+            <build>
+                <plugins>
+                    
+                    <plugin>
+                        <artifactId>maven-enforcer-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <goals>
+                                    <goal>enforce</goal>
+                                </goals>
+                                <configuration>
+                                    <rules>
+                                        <requireProperty>
+                                            <property>cache.server</property>
+                                            <property>cache.server.jboss.groupId</property>
+                                            <property>cache.server.jboss.artifactId</property>
+                                            <property>cache.server.jboss.version</property>
+                                            <property>cache.server.jboss.unpacked.folder.name</property>
+                                        </requireProperty>
+                                    </rules>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                    <plugin>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>unpack-cache-server</id>
+                                <phase>generate-resources</phase>
+                                <goals>
+                                    <goal>unpack</goal>
+                                </goals>
+                                <configuration>
+                                    <artifactItems>
+                                        <artifactItem>
+                                            <groupId>${cache.server.jboss.groupId}</groupId>
+                                            <artifactId>${cache.server.jboss.artifactId}</artifactId>
+                                            <version>${cache.server.jboss.version}</version>
+                                            <type>zip</type>
+                                            <classifier>bin</classifier>
+                                            <outputDirectory>${containers.home}</outputDirectory>
+                                        </artifactItem>
+                                    </artifactItems>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+            
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>xml-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>configure-adapter-debug-log</id>
+                                <phase>process-test-resources</phase>
+                                <goals>
+                                    <goal>transform</goal>
+                                </goals>
+                                <configuration>
+                                    <transformationSets>
+                                        <transformationSet>
+                                            <dir>${cache.server.jboss.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone.xml</include>
+                                            </includes>
+                                            <stylesheet>${common.resources}/add-keycloak-remote-store.xsl</stylesheet>
+                                            <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
+                                        </transformationSet>
+                                    </transformationSets>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>io-worker-threads</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>transform</goal>
+                                </goals>
+                                <configuration>
+                                    <transformationSets>
+                                        <transformationSet>
+                                            <dir>${cache.server.jboss.home}/standalone/configuration</dir>
+                                            <includes>
+                                                <include>standalone.xml</include>
+                                                <include>standalone-ha.xml</include>
+                                            </includes>
+                                            <stylesheet>${common.resources}/io.xsl</stylesheet>
+                                            <outputDir>${cache.server.jboss.home}/standalone/configuration</outputDir>
+                                            <parameters>
+                                                <parameter>
+                                                    <name>worker.io-threads</name>
+                                                    <value>${cache.server.worker.io-threads}</value>
+                                                </parameter>
+                                                <parameter>
+                                                    <name>worker.task-max-threads</name>
+                                                    <value>${cache.server.worker.task-max-threads}</value>
+                                                </parameter>
+                                            </parameters>
+                                        </transformationSet>
+                                    </transformationSets>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                    <plugin>
+                        <artifactId>maven-resources-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>enable-jboss-mgmt-admin</id>
+                                <phase>process-resources</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${cache.server.jboss.home}/standalone/configuration</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${common.resources}</directory>
+                                            <includes>
+                                                <include>mgmt-users.properties</include>
+                                            </includes>
+                                        </resource>
+                                    </resources>
+                                    <overwrite>true</overwrite>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>create-zip</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <descriptors>
+                                        <descriptor>${assembly.xml}</descriptor>
+                                    </descriptors>
+                                    <appendAssemblyId>false</appendAssemblyId>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    
+                </plugins>
+            </build>
+        </profile>
+        
+        <profile>
+            <id>cache-server-infinispan</id>
+            <modules>
+                <module>infinispan</module>
+            </modules>
+        </profile>
+    </profiles>
+
+</project>

testsuite/integration-arquillian/servers/cache-server/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+~ Copyright 2016 Red Hat, Inc. and/or its affiliates
+~ and other contributors as indicated by the @author tags.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <parent>
+        <groupId>org.keycloak.testsuite</groupId>
+        <artifactId>integration-arquillian-servers</artifactId>
+        <version>3.2.0.CR1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>integration-arquillian-servers-cache-server</artifactId>
+    <packaging>pom</packaging>
+    <name>Cache Server</name>
+
+    <properties>
+        <auth.server.worker.io-threads>${jboss.default.worker.io-threads}</auth.server.worker.io-threads>
+        <auth.server.worker.task-max-threads>${jboss.default.worker.task-max-threads}</auth.server.worker.task-max-threads>
+    </properties>
+
+    <modules>
+        <module>jboss</module>
+    </modules>
+
+</project>

testsuite/integration-arquillian/servers/pom.xml

@@ -46,13 +46,20 @@
         <!--<fuse62.version>6.2.0.redhat-133</fuse62.version>-->
         <fuse62.version>6.2.1.redhat-084</fuse62.version>
         
+        <!-- cache server versions -->
+        <infinispan.version>9.0.1.Final</infinispan.version>
+        
         <jboss.default.worker.io-threads>16</jboss.default.worker.io-threads>
         <jboss.default.worker.task-max-threads>128</jboss.default.worker.task-max-threads>
+
+        <cache.default.worker.io-threads>2</cache.default.worker.io-threads>
+        <cache.default.worker.task-max-threads>4</cache.default.worker.task-max-threads>
     </properties>
 
     <modules>
         <module>auth-server</module>
         <module>app-server</module>
+        <module>cache-server</module>
     </modules>
 
     <profiles>

testsuite/integration-arquillian/tests/base/pom.xml

@@ -40,6 +40,7 @@
         <exclude.client>-</exclude.client>
         <!--exclude cluster tests by default, enabled by 'auth-server-*-cluster' profiles in tests/pom.xml-->
         <exclude.cluster>**/cluster/**/*Test.java</exclude.cluster>
+        <exclude.crossdc>**/crossdc/**/*Test.java</exclude.crossdc>
         <!-- exclude x509 tests by default, enabled by 'ssl' profile -->
         <exclude.x509>**/x509/*Test.java</exclude.x509>
         <!-- exclude undertow adapter tests. They can be added by -Dtest=org.keycloak.testsuite.adapter.undertow.**.*Test -->
@@ -125,6 +126,7 @@
                         <exclude>${exclude.account}</exclude>
                         <exclude>${exclude.client}</exclude>
                         <exclude>${exclude.cluster}</exclude>
+                        <exclude>${exclude.crossdc</exclude>
                         <exclude>${exclude.undertow.adapter}</exclude>
                         <exclude>${exclude.x509}</exclude>
                     </excludes>

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/AuthServerTestEnricher.java

@@ -16,7 +16,6 @@
  */
 package org.keycloak.testsuite.arquillian;
 
-import org.jboss.arquillian.container.spi.Container;
 import org.jboss.arquillian.container.spi.ContainerRegistry;
 import org.jboss.arquillian.container.spi.event.StartContainer;
 import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
@@ -41,10 +40,11 @@ import org.keycloak.testsuite.util.OAuthClient;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
+import java.util.stream.Collectors;
 import javax.ws.rs.NotFoundException;
 
 /**
@@ -68,8 +68,18 @@ public class AuthServerTestEnricher {
     private static final String AUTH_SERVER_CONTAINER_PROPERTY = "auth.server.container";
     public static final String AUTH_SERVER_CONTAINER = System.getProperty(AUTH_SERVER_CONTAINER_PROPERTY, AUTH_SERVER_CONTAINER_DEFAULT);
 
+    private static final String AUTH_SERVER_BACKEND_DEFAULT = AUTH_SERVER_CONTAINER + "-backend";
+    private static final String AUTH_SERVER_BACKEND_PROPERTY = "auth.server.backend";
+    public static final String AUTH_SERVER_BACKEND = System.getProperty(AUTH_SERVER_BACKEND_PROPERTY, AUTH_SERVER_BACKEND_DEFAULT);
+
+    private static final String AUTH_SERVER_BALANCER_DEFAULT = "auth-server-balancer";
+    private static final String AUTH_SERVER_BALANCER_PROPERTY = "auth.server.balancer";
+    public static final String AUTH_SERVER_BALANCER = System.getProperty(AUTH_SERVER_BALANCER_PROPERTY, AUTH_SERVER_BALANCER_DEFAULT);
+
     private static final String AUTH_SERVER_CLUSTER_PROPERTY = "auth.server.cluster";
     public static final boolean AUTH_SERVER_CLUSTER = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CLUSTER_PROPERTY, "false"));
+    private static final String AUTH_SERVER_CROSS_DC_PROPERTY = "auth.server.crossdc";
+    public static final boolean AUTH_SERVER_CROSS_DC = Boolean.parseBoolean(System.getProperty(AUTH_SERVER_CROSS_DC_PROPERTY, "false"));
 
     private static final boolean AUTH_SERVER_UNDERTOW_CLUSTER = Boolean.parseBoolean(System.getProperty("auth.server.undertow.cluster", "false"));
 
@@ -106,46 +116,82 @@ public class AuthServerTestEnricher {
     }
 
     public void initializeSuiteContext(@Observes(precedence = 2) BeforeSuite event) {
-
+        Set<ContainerInfo> containers = containerRegistry.get().getContainers().stream()
-        Set<ContainerInfo> containers = new LinkedHashSet<>();
+          .map(ContainerInfo::new)
-        for (Container c : containerRegistry.get().getContainers()) {
+          .collect(Collectors.toSet());
-            containers.add(new ContainerInfo(c));
-        }
 
         suiteContext = new SuiteContext(containers);
 
-        String authServerFrontend = null;
+        if (AUTH_SERVER_CROSS_DC) {
+            // if cross-dc mode enabled, load-balancer is the frontend of datacenter cluster
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
+              });
 
-        if (AUTH_SERVER_CLUSTER) {
+            if (suiteContext.getDcAuthServerInfo().isEmpty()) {
+                throw new IllegalStateException("Not found frontend container (load balancer): " + AUTH_SERVER_BALANCER);
+            }
+            if (suiteContext.getDcAuthServerInfo().stream().anyMatch(Objects::isNull)) {
+                throw new IllegalStateException("Frontend container (load balancer) misconfiguration");
+            }
+
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER + "-cross-dc-"))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerBackendsInfo(Integer.valueOf(dcString), c);
+              });
+
+            if (suiteContext.getDcAuthServerInfo().isEmpty()) {
+                throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", AUTH_SERVER_BACKEND));
+            }
+            if (suiteContext.getDcAuthServerBackendsInfo().stream().anyMatch(Objects::isNull)) {
+                throw new IllegalStateException("Frontend container (load balancer) misconfiguration");
+            }
+            if (suiteContext.getDcAuthServerBackendsInfo().stream().anyMatch(List::isEmpty)) {
+                throw new RuntimeException(String.format("Some data center has no auth server container matching '%s' defined in arquillian.xml.", AUTH_SERVER_BACKEND));
+            }
+
+            log.info("Using frontend containers: " + this.suiteContext.getDcAuthServerInfo().stream()
+              .map(ContainerInfo::getQualifier)
+              .collect(Collectors.joining(", ")));
+        } else if (AUTH_SERVER_CLUSTER) {
             // if cluster mode enabled, load-balancer is the frontend
-            for (ContainerInfo c : containers) {
+            ContainerInfo container = containers.stream()
-                if (c.getQualifier().startsWith("auth-server-balancer")) {
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER))
-                    authServerFrontend = c.getQualifier();
+              .findAny()
-                }
+              .orElseThrow(() -> new IllegalStateException("Not found frontend container: " + AUTH_SERVER_BALANCER));
+            updateWithAuthServerInfo(container);
+            suiteContext.setAuthServerInfo(container);
+
+            containers.stream()
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BACKEND))
+              .forEach(c -> {
+                String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
+                updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
+                suiteContext.addAuthServerBackendsInfo(0, c);
+              });
+
+            if (suiteContext.getAuthServerBackendsInfo().isEmpty()) {
+                throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", AUTH_SERVER_BACKEND));
             }
 
-            if (authServerFrontend != null) {
+            log.info("Using frontend container: " + container.getQualifier());
-                log.info("Using frontend container: " + authServerFrontend);
-            } else {
-                throw new IllegalStateException("Not found frontend container");
-            }
         } else {
-            authServerFrontend = AUTH_SERVER_CONTAINER; // single-node mode
+            // frontend-only
-        }
+            ContainerInfo container = containers.stream()
-
+              .filter(c -> c.getQualifier().startsWith(AUTH_SERVER_CONTAINER))
-        String authServerBackend = AUTH_SERVER_CONTAINER + "-backend";
+              .findAny()
-        int backends = 0;
+              .orElseThrow(() -> new IllegalStateException("Not found frontend container: " + AUTH_SERVER_CONTAINER));
-        for (ContainerInfo container : suiteContext.getContainers()) {
+            updateWithAuthServerInfo(container);
-            // frontend
+            suiteContext.setAuthServerInfo(container);
-            if (container.getQualifier().equals(authServerFrontend)) {
-                updateWithAuthServerInfo(container);
-                suiteContext.setAuthServerInfo(container);
-            }
-            // backends
-            if (AUTH_SERVER_CLUSTER && container.getQualifier().startsWith(authServerBackend)) {
-                updateWithAuthServerInfo(container, ++backends);
-                suiteContext.getAuthServerBackendsInfo().add(container);
-            }
         }
 
         // Setup with 2 undertow backend nodes and no loadbalancer.
@@ -153,14 +199,6 @@ public class AuthServerTestEnricher {
 //            suiteContext.setAuthServerInfo(suiteContext.getAuthServerBackendsInfo().get(0));
 //        }
 
-        // validate auth server setup
-        if (suiteContext.getAuthServerInfo() == null) {
-            throw new RuntimeException(String.format("No auth server container matching '%s' found in arquillian.xml.", authServerFrontend));
-        }
-        if (AUTH_SERVER_CLUSTER && suiteContext.getAuthServerBackendsInfo().isEmpty()) {
-            throw new RuntimeException(String.format("No auth server container matching '%sN' found in arquillian.xml.", authServerBackend));
-        }
-
         if (START_MIGRATION_CONTAINER) {
             // init migratedAuthServerInfo
             for (ContainerInfo container : suiteContext.getContainers()) {

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/ContainerInfo.java

@@ -5,6 +5,8 @@ import org.jboss.arquillian.container.spi.Container;
 import java.net.URL;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Stream;
+import org.jboss.arquillian.container.spi.Container.State;
 
 /**
  *
@@ -97,4 +99,12 @@ public class ContainerInfo {
                 other.arquillianContainer.getContainerConfiguration().getContainerName());
     }
 
+    public boolean isStarted() {
+        return arquillianContainer.getState() == State.STARTED;
+    }
+
+    public boolean isManual() {
+        return Objects.equals(arquillianContainer.getContainerConfiguration().getMode(), "manual");
+    }
+
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/KeycloakArquillianExtension.java

@@ -35,6 +35,7 @@ import org.keycloak.testsuite.arquillian.h2.H2TestEnricher;
 import org.keycloak.testsuite.arquillian.karaf.CustomKarafContainer;
 import org.keycloak.testsuite.arquillian.migration.MigrationTestExecutionDecider;
 import org.keycloak.testsuite.arquillian.provider.AdminClientProvider;
+import org.keycloak.testsuite.arquillian.provider.LoadBalancerControllerProvider;
 import org.keycloak.testsuite.arquillian.provider.OAuthClientProvider;
 import org.keycloak.testsuite.arquillian.provider.SuiteContextProvider;
 import org.keycloak.testsuite.arquillian.provider.TestContextProvider;
@@ -57,7 +58,8 @@ public class KeycloakArquillianExtension implements LoadableExtension {
                 .service(ResourceProvider.class, SuiteContextProvider.class)
                 .service(ResourceProvider.class, TestContextProvider.class)
                 .service(ResourceProvider.class, AdminClientProvider.class)
-                .service(ResourceProvider.class, OAuthClientProvider.class);
+                .service(ResourceProvider.class, OAuthClientProvider.class)
+                .service(ResourceProvider.class, LoadBalancerControllerProvider.class);
 
         builder
                 .service(DeploymentScenarioGenerator.class, DeploymentTargetModifier.class)

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/SuiteContext.java

@@ -24,6 +24,7 @@ import java.util.Set;
 
 import org.keycloak.testsuite.arquillian.migration.MigrationContext;
 
+import java.util.LinkedList;
 import static org.keycloak.testsuite.util.MailServerConfiguration.FROM;
 import static org.keycloak.testsuite.util.MailServerConfiguration.HOST;
 import static org.keycloak.testsuite.util.MailServerConfiguration.PORT;
@@ -36,8 +37,8 @@ public final class SuiteContext {
 
     private final Set<ContainerInfo> container;
 
-    private ContainerInfo authServerInfo;
+    private List<ContainerInfo> authServerInfo = new LinkedList<>();
-    private final List<ContainerInfo> authServerBackendsInfo = new ArrayList<>();
+    private final List<List<ContainerInfo>> authServerBackendsInfo = new ArrayList<>();
 
     private ContainerInfo migratedAuthServerInfo;
     private final MigrationContext migrationContext = new MigrationContext();
@@ -72,17 +73,48 @@ public final class SuiteContext {
     }
 
     public ContainerInfo getAuthServerInfo() {
+        return getAuthServerInfo(0);
+    }
+
+    public ContainerInfo getAuthServerInfo(int dcIndex) {
+        return authServerInfo.get(dcIndex);
+    }
+
+    public List<ContainerInfo> getDcAuthServerInfo() {
         return authServerInfo;
     }
 
     public void setAuthServerInfo(ContainerInfo authServerInfo) {
-        this.authServerInfo = authServerInfo;
+        this.authServerInfo = new LinkedList<>();
+        this.authServerInfo.add(authServerInfo);
+    }
+
+    public void addAuthServerInfo(int dcIndex, ContainerInfo serverInfo) {
+        while (dcIndex >= authServerInfo.size()) {
+            authServerInfo.add(null);
+        }
+        this.authServerInfo.set(dcIndex, serverInfo);
     }
 
     public List<ContainerInfo> getAuthServerBackendsInfo() {
+        return getAuthServerBackendsInfo(0);
+    }
+
+    public List<ContainerInfo> getAuthServerBackendsInfo(int dcIndex) {
+        return authServerBackendsInfo.get(dcIndex);
+    }
+
+    public List<List<ContainerInfo>> getDcAuthServerBackendsInfo() {
         return authServerBackendsInfo;
     }
 
+    public void addAuthServerBackendsInfo(int dcIndex, ContainerInfo container) {
+        while (dcIndex >= authServerBackendsInfo.size()) {
+            authServerBackendsInfo.add(new LinkedList<>());
+        }
+        authServerBackendsInfo.get(dcIndex).add(container);
+    }
+
     public ContainerInfo getMigratedAuthServerInfo() {
         return migratedAuthServerInfo;
     }
@@ -96,7 +128,11 @@ public final class SuiteContext {
     }
 
     public boolean isAuthServerCluster() {
-        return !authServerBackendsInfo.isEmpty();
+        return ! authServerBackendsInfo.isEmpty();
+    }
+
+    public boolean isAuthServerCrossDc() {
+        return authServerBackendsInfo.size() > 1;
     }
 
     public boolean isAuthServerMigrationEnabled() {
@@ -113,19 +149,38 @@ public final class SuiteContext {
 
     @Override
     public String toString() {
-        String containers = "Auth server: " + (isAuthServerCluster() ? "\nFrontend: " : "")
+        StringBuilder sb = new StringBuilder("SUITE CONTEXT:\nAuth server: ");
-                + authServerInfo.getQualifier() + "\n";
+
-        for (ContainerInfo bInfo : getAuthServerBackendsInfo()) {
+        if (isAuthServerCrossDc()) {
-            containers += "Backend: " + bInfo + "\n";
+            for (int i = 0; i < authServerInfo.size(); i ++) {
+                ContainerInfo frontend = this.authServerInfo.get(i);
+                sb.append("\nFrontend (dc=").append(i).append("): ").append(frontend.getQualifier()).append("\n");
+            }
+
+            for (int i = 0; i < authServerBackendsInfo.size(); i ++) {
+                int dcIndex = i;
+                getDcAuthServerBackendsInfo().get(i).forEach(bInfo -> sb.append("Backend (dc=").append(dcIndex).append("): ").append(bInfo).append("\n"));
+            }
+        } else if (isAuthServerCluster()) {
+            sb.append(isAuthServerCluster() ? "\nFrontend: " : "")
+              .append(getAuthServerInfo().getQualifier())
+              .append("\n");
+
+            getAuthServerBackendsInfo().forEach(bInfo -> sb.append("  Backend: ").append(bInfo).append("\n"));
+        } else {
+          sb.append(getAuthServerInfo().getQualifier())
+            .append("\n");
         }
+
+
         if (isAuthServerMigrationEnabled()) {
-            containers += "Migrated from: " + System.getProperty("migrated.auth.server.version") + "\n";
+            sb.append("Migrated from: ").append(System.getProperty("migrated.auth.server.version")).append("\n");
         }
+
         if (isAdapterCompatTesting()) {
-            containers += "Adapter backward compatibility testing mode!\n";
+            sb.append("Adapter backward compatibility testing mode!\n");
         }
-        return "SUITE CONTEXT:\n"
+        return sb.toString();
-                + containers;
     }
 
 }

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/annotation/LoadBalancer.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.arquillian.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author hmlnarik
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.FIELD })
+public @interface LoadBalancer {
+    String value() default "";
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/RegistryCreator.java

@@ -110,6 +110,8 @@ public class RegistryCreator {
             if (isClassPresent(getAdapterImplClassValue(containerDef))) {
                 return DeployableContainer.class.isAssignableFrom(
                         loadClass(getAdapterImplClassValue(containerDef)));
+            } else {
+                log.warn("Cannot load adapterImpl class for " + containerDef.getContainerName());
             }
         }
 

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/provider/LoadBalancerControllerProvider.java

@@ -0,0 +1,67 @@
+package org.keycloak.testsuite.arquillian.provider;
+
+import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
+import java.lang.annotation.Annotation;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.jboss.arquillian.container.spi.event.KillContainer;
+import org.jboss.arquillian.container.spi.event.StartContainer;
+import org.jboss.arquillian.container.spi.event.StopContainer;
+import org.jboss.arquillian.core.api.Instance;
+import org.jboss.arquillian.core.api.annotation.Inject;
+import org.jboss.arquillian.core.api.annotation.Observes;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.jboss.arquillian.test.spi.enricher.resource.ResourceProvider;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
+import org.jboss.arquillian.container.spi.Container;
+import org.jboss.arquillian.container.spi.ContainerRegistry;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class LoadBalancerControllerProvider implements ResourceProvider {
+
+    @Inject
+    private Instance<ContainerRegistry> registry;
+
+    @Override
+    public boolean canProvide(Class<?> type) {
+        return type.equals(LoadBalancerController.class);
+    }
+
+    @Override
+    public Object lookup(ArquillianResource resource, Annotation... qualifiers) {
+        String balancerName = null;
+
+        // Check for the presence of possible qualifiers
+        for (Annotation a : qualifiers) {
+            Class<? extends Annotation> annotationType = a.annotationType();
+
+            if (annotationType.equals(LoadBalancer.class)) {
+                balancerName = ((LoadBalancer) a).value();
+            }
+        }
+
+        ContainerRegistry reg = registry.get();
+        Container container = null;
+        if (balancerName == null || "".equals(balancerName.trim())) {
+            if (reg.getContainers().size() == 1) {
+                container = reg.getContainers().get(0);
+            } else {
+                throw new IllegalArgumentException("Invalid load balancer configuration request - need to specify load balancer name in @LoadBalancerController");
+            }
+        } else {
+            container = reg.getContainer(balancerName);
+        }
+
+        if (container == null) {
+            throw new IllegalArgumentException("Invalid load balancer configuration - load balancer not found: '" + balancerName + "'");
+        }
+        if (! (container.getDeployableContainer() instanceof LoadBalancerController)) {
+            throw new IllegalArgumentException("Invalid load balancer configuration - container " + container.getName() + " is not a load balancer");
+        }
+
+        return container.getDeployableContainer();
+    }
+}

testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/util/SetSystemProperty.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.testsuite.util;
+
+import java.io.Closeable;
+
+/**
+ * @author <a href="mailto:mposolda@redhat.com">Marek Posolda</a>
+ */
+public class SetSystemProperty implements Closeable {
+
+    private final String name;
+    private final String oldValue;
+
+    public SetSystemProperty(String name, String value) {
+        this.name = name;
+        this.oldValue = System.getProperty(name);
+
+        if (value == null) {
+            if (oldValue != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, value);
+        }
+    }
+
+    public void revert() {
+        String value = System.getProperty(name);
+
+        if (oldValue == null) {
+            if (value != null) {
+                System.getProperties().remove(name);
+            }
+        } else {
+            System.setProperty(name, oldValue);
+        }
+    }
+
+    @Override
+    public void close() {
+        revert();
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractAdminCrossDCTest.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.resource.RealmResource;
+import org.keycloak.events.log.JBossLoggingEventListenerProviderFactory;
+import org.keycloak.representations.idm.RealmRepresentation;
+import org.keycloak.testsuite.events.EventsListenerProviderFactory;
+import org.keycloak.testsuite.util.TestCleanup;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.Before;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public abstract class AbstractAdminCrossDCTest extends AbstractCrossDCTest {
+
+    protected static final String REALM_NAME = "admin-client-test";
+
+    protected RealmResource realm;
+    protected String realmId;
+
+
+    @Override
+    public void configureTestRealm(RealmRepresentation testRealm) {
+        findTestApp(testRealm).setDirectAccessGrantsEnabled(true);
+    }
+
+
+
+    @Override
+    public void addTestRealms(List<RealmRepresentation> testRealms) {
+        super.addTestRealms(testRealms);
+
+        RealmRepresentation adminRealmRep = new RealmRepresentation();
+        adminRealmRep.setId(REALM_NAME);
+        adminRealmRep.setRealm(REALM_NAME);
+        adminRealmRep.setEnabled(true);
+        Map<String, String> config = new HashMap<>();
+        config.put("from", "auto@keycloak.org");
+        config.put("host", "localhost");
+        config.put("port", "3025");
+        adminRealmRep.setSmtpServer(config);
+
+        List<String> eventListeners = new ArrayList<>();
+        eventListeners.add(JBossLoggingEventListenerProviderFactory.ID);
+        eventListeners.add(EventsListenerProviderFactory.PROVIDER_ID);
+        adminRealmRep.setEventsListeners(eventListeners);
+
+        testRealms.add(adminRealmRep);
+    }
+
+    @Before
+    public void setRealm() {
+        realm = adminClient.realm(REALM_NAME);
+        realmId = realm.toRepresentation().getId();
+    }
+
+    @Override
+    protected TestCleanup getCleanup() {
+        return getCleanup(REALM_NAME);
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/AbstractCrossDCTest.java

@@ -0,0 +1,161 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.models.Constants;
+import org.keycloak.testsuite.AbstractTestRealmKeycloakTest;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.arquillian.LoadBalancerController;
+import org.keycloak.testsuite.arquillian.annotation.LoadBalancer;
+import org.keycloak.testsuite.auth.page.AuthRealm;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.jboss.arquillian.container.test.api.ContainerController;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public abstract class AbstractCrossDCTest extends AbstractTestRealmKeycloakTest {
+
+    @ArquillianResource
+    @LoadBalancer(value = "auth-server-balancer-cross-dc")
+    protected LoadBalancerController loadBalancerCtrl;
+
+    @ArquillianResource
+    protected ContainerController containerController;
+
+    protected Map<ContainerInfo, Keycloak> backendAdminClients = new HashMap<>();
+
+    @After
+    @Before
+    public void enableOnlyFirstNodeInFirstDc() {
+        this.loadBalancerCtrl.disableAllBackendNodes();
+        loadBalancerCtrl.enableBackendNodeByName(getAutomaticallyStartedBackendNodes(0)
+          .findFirst()
+          .orElseThrow(() -> new IllegalStateException("No node is started automatically"))
+          .getQualifier()
+        );
+    }
+
+    @Before
+    public void terminateManuallyStartedServers() {
+        log.debug("Halting all nodes that are started manually");
+        this.suiteContext.getDcAuthServerBackendsInfo().stream()
+          .flatMap(List::stream)
+          .filter(ContainerInfo::isStarted)
+          .filter(ContainerInfo::isManual)
+          .map(ContainerInfo::getQualifier)
+          .forEach(containerController::stop);
+    }
+
+    @Override
+    public void importTestRealms() {
+        enableOnlyFirstNodeInFirstDc();
+        super.importTestRealms();
+    }
+
+    @Override
+    public void afterAbstractKeycloakTest() {
+        enableOnlyFirstNodeInFirstDc();
+        super.afterAbstractKeycloakTest();
+    }
+
+    @Override
+    public void deleteCookies() {
+        enableOnlyFirstNodeInFirstDc();
+        super.deleteCookies();
+    }
+
+    @Before
+    public void initLoadBalancer() {
+        log.debug("Initializing load balancer - only enabling started nodes in the first DC");
+        this.loadBalancerCtrl.disableAllBackendNodes();
+        // Enable only the started nodes in each datacenter
+        this.suiteContext.getDcAuthServerBackendsInfo().get(0).stream()
+          .filter(ContainerInfo::isStarted)
+          .map(ContainerInfo::getQualifier)
+          .forEach(loadBalancerCtrl::enableBackendNodeByName);
+    }
+
+    protected Keycloak createAdminClientFor(ContainerInfo node) {
+        log.info("Initializing admin client for " + node.getContextRoot() + "/auth");
+        return Keycloak.getInstance(node.getContextRoot() + "/auth", AuthRealm.MASTER, AuthRealm.ADMIN, AuthRealm.ADMIN, Constants.ADMIN_CLI_CLIENT_ID);
+    }
+
+    protected Keycloak getAdminClientFor(ContainerInfo node) {
+        Keycloak adminClient = backendAdminClients.get(node);
+        if (adminClient == null && node.equals(suiteContext.getAuthServerInfo())) {
+            adminClient = this.adminClient;
+        }
+        return adminClient;
+    }
+
+    public void disableDcOnLoadBalancer(int dcIndex) {
+        log.infof("Disabling load balancer for dc=%d", dcIndex);
+        this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).forEach(c -> loadBalancerCtrl.disableBackendNodeByName(c.getQualifier()));
+    }
+
+    /**
+     * Enables all started nodes in the given data center
+     * @param dcIndex
+     */
+    public void enableDcOnLoadBalancer(int dcIndex) {
+        log.infof("Enabling load balancer for dc=%d", dcIndex);
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        if (! dcNodes.stream().anyMatch(ContainerInfo::isStarted)) {
+            log.warnf("No node is started in DC %d", dcIndex);
+        } else {
+            dcNodes.stream()
+              .filter(ContainerInfo::isStarted)
+              .forEach(c -> loadBalancerCtrl.enableBackendNodeByName(c.getQualifier()));
+        }
+    }
+
+    public void disableLoadBalancerNode(int dcIndex, int nodeIndex) {
+        log.infof("Disabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        loadBalancerCtrl.disableBackendNodeByName(this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex).getQualifier());
+    }
+
+    public void enableLoadBalancerNode(int dcIndex, int nodeIndex) {
+        log.infof("Enabling load balancer for dc=%d, node=%d", dcIndex, nodeIndex);
+        final ContainerInfo backendNode = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex).get(nodeIndex);
+        if (backendNode == null) {
+            throw new IllegalArgumentException("Invalid node with index " + nodeIndex + " for DC " + dcIndex);
+        }
+        if (! backendNode.isStarted()) {
+            log.warnf("Node %s is not started in DC %d", backendNode.getQualifier(), dcIndex);
+        }
+        loadBalancerCtrl.enableBackendNodeByName(backendNode.getQualifier());
+    }
+
+    public Stream<ContainerInfo> getManuallyStartedBackendNodes(int dcIndex) {
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        return dcNodes.stream().filter(ContainerInfo::isManual);
+    }
+
+    public Stream<ContainerInfo> getAutomaticallyStartedBackendNodes(int dcIndex) {
+        final List<ContainerInfo> dcNodes = this.suiteContext.getDcAuthServerBackendsInfo().get(dcIndex);
+        return dcNodes.stream().filter(c -> ! c.isManual());
+    }
+}

testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/ActionTokenCrossDCTest.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2017 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.keycloak.testsuite.crossdc;
+
+import org.keycloak.admin.client.resource.UserResource;
+import org.keycloak.events.admin.OperationType;
+import org.keycloak.events.admin.ResourceType;
+import org.keycloak.models.UserModel;
+import org.keycloak.representations.idm.UserRepresentation;
+import org.keycloak.testsuite.Retry;
+import org.keycloak.testsuite.admin.ApiUtil;
+import org.keycloak.testsuite.arquillian.ContainerInfo;
+import org.keycloak.testsuite.page.LoginPasswordUpdatePage;
+import org.keycloak.testsuite.pages.ErrorPage;
+import org.keycloak.testsuite.util.AdminEventPaths;
+import org.keycloak.testsuite.util.GreenMailRule;
+import org.keycloak.testsuite.util.MailUtils;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import javax.mail.MessagingException;
+import javax.mail.internet.MimeMessage;
+import javax.ws.rs.core.Response;
+import org.jboss.arquillian.graphene.page.Page;
+import org.jboss.arquillian.test.api.ArquillianResource;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+/**
+ *
+ * @author hmlnarik
+ */
+public class ActionTokenCrossDCTest extends AbstractAdminCrossDCTest {
+
+    @Rule
+    public GreenMailRule greenMail = new GreenMailRule();
+
+    @Page
+    protected LoginPasswordUpdatePage passwordUpdatePage;
+
+    @Page
+    protected ErrorPage errorPage;
+
+    private String createUser(UserRepresentation userRep) {
+        Response response = realm.users().create(userRep);
+        String createdId = ApiUtil.getCreatedId(response);
+        response.close();
+
+        getCleanup().addUserId(createdId);
+
+        return createdId;
+    }
+
+    @Test
+    public void sendResetPasswordEmailSuccessWorksInCrossDc() throws IOException, MessagingException {
+        UserRepresentation userRep = new UserRepresentation();
+        userRep.setEnabled(true);
+        userRep.setUsername("user1");
+        userRep.setEmail("user1@test.com");
+
+        String id = createUser(userRep);
+
+        UserResource user = realm.users().get(id);
+        List<String> actions = new LinkedList<>();
+        actions.add(UserModel.RequiredAction.UPDATE_PASSWORD.name());
+        user.executeActionsEmail(actions);
+
+        Assert.assertEquals(1, greenMail.getReceivedMessages().length);
+
+        MimeMessage message = greenMail.getReceivedMessages()[0];
+
+        String link = MailUtils.getPasswordResetEmailLink(message);
+
+        driver.navigate().to(link);
+
+        passwordUpdatePage.assertCurrent();
+
+        passwordUpdatePage.changePassword("new-pass", "new-pass");
+
+        assertEquals("Your account has been updated.", driver.getTitle());
+
+        disableDcOnLoadBalancer(0);
+        enableDcOnLoadBalancer(1);
+
+        Retry.execute(() -> {
+            driver.navigate().to(link);
+            errorPage.assertCurrent();
+        }, 3, 400);
+    }
+
+    @Ignore("KEYCLOAK-5030")
+    @Test
+    public void sendResetPasswordEmailAfterNewNodeAdded() throws IOException, MessagingException {
+        disableDcOnLoadBalancer(1);
+
+        UserRepresentation userRep = new UserRepresentation();
+        userRep.setEnabled(true);
+        userRep.setUsername("user1");
+        userRep.setEmail("user1@test.com");
+
+        String id = createUser(userRep);
+
+        UserResource user = realm.users().get(id);
+        List<String> actions = new LinkedList<>();
+        actions.add(UserModel.RequiredAction.UPDATE_PASSWORD.name());
+        user.executeActionsEmail(actions);
+
+        Assert.assertEquals(1, greenMail.getReceivedMessages().length);
+
+        MimeMessage message = greenMail.getReceivedMessages()[0];
+
+        String link = MailUtils.getPasswordResetEmailLink(message);
+
+        driver.navigate().to(link);
+
+        passwordUpdatePage.assertCurrent();
+
+        passwordUpdatePage.changePassword("new-pass", "new-pass");
+
+        assertEquals("Your account has been updated.", driver.getTitle());
+
+        disableDcOnLoadBalancer(0);
+        getManuallyStartedBackendNodes(1)
+          .findFirst()
+          .ifPresent(c -> {
+              containerController.start(c.getQualifier());
+              loadBalancerCtrl.enableBackendNodeByName(c.getQualifier());
+          });
+
+        driver.navigate().to(link);
+
+        errorPage.assertCurrent();
+    }
+
+}

testsuite/integration-arquillian/tests/base/src/test/resources/META-INF/keycloak-server.json

@@ -107,16 +107,24 @@
 
     "connectionsInfinispan": {
         "default": {
+            "jgroupsUdpMcastAddr": "${keycloak.connectionsInfinispan.jgroupsUdpMcastAddr:234.56.78.90}",
+            "nodeName": "${keycloak.connectionsInfinispan.nodeName,jboss.node.name:defaultNodeName}",
             "clustered": "${keycloak.connectionsInfinispan.clustered:false}",
             "async": "${keycloak.connectionsInfinispan.async:false}",
             "sessionsOwners": "${keycloak.connectionsInfinispan.sessionsOwners:1}",
             "l1Lifespan": "${keycloak.connectionsInfinispan.l1Lifespan:600000}",
             "remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:false}",
-            "remoteStoreHost": "${keycloak.connectionsInfinispan.remoteStoreHost:localhost}",
+            "remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
             "remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}"
         }
     },
     
+    "stickySessionEncoder": {
+        "infinispan": {
+            "nodeName": "${keycloak.stickySessionEncoder.nodeName,jboss.node.name:defaultNodeName}"
+        }
+    },
+
 
     "truststore": {
         "file": {

testsuite/integration-arquillian/tests/base/src/test/resources/arquillian.xml

@@ -103,6 +103,7 @@
                 <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend1.management.port}</property>
                 <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+                <property name="bindHttpPortOffset">${auth.server.backend1.port.offset}</property>
             </configuration>
         </container>
         <container qualifier="auth-server-${auth.server}-backend2" mode="manual" >
@@ -124,6 +125,7 @@
                 <property name="outputToConsole">${backends.console.output}</property>
                 <property name="managementPort">${auth.server.backend2.management.port}</property>
                 <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+                <property name="bindHttpPortOffset">${auth.server.backend2.port.offset}</property>
             </configuration>
         </container>
     </group>
@@ -165,6 +167,138 @@
     </group>
 
 
+    <!-- Cross DC with embedded undertow. Node numbering is [centre #].[node #] -->
+    <group qualifier="auth-server-undertow-cross-dc">
+        <container qualifier="cache-server-cross-dc" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.jboss.as.arquillian.container.managed.ManagedDeployableContainer</property>
+                <property name="jbossHome">${cache.server.home}</property>
+                <property name="serverConfig">standalone.xml</property>
+                <property name="jbossArguments">
+                    -Djboss.socket.binding.port-offset=${cache.server.port.offset}
+                    -Djboss.default.multicast.address=234.56.78.99
+                    -Djboss.node.name=cache-server
+                    ${adapter.test.props}
+                    ${auth.server.profile}
+                </property>
+                <property name="javaVmArguments">
+                    ${auth.server.memory.settings}
+                    -Djava.net.preferIPv4Stack=true
+                </property>
+                <property name="outputToConsole">${cache.server.console.output}</property>
+                <property name="managementPort">${cache.server.management.port}</property>
+                <property name="startupTimeoutInSeconds">${auth.server.jboss.startup.timeout}</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-balancer-cross-dc" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.lb.SimpleUndertowLoadBalancerContainer</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">5</property>
+                <property name="nodes">auth-server-undertow-cross-dc-0.1=http://localhost:8101,auth-server-undertow-cross-dc-0.2-manual=http://localhost:8102,auth-server-undertow-cross-dc-1.1=http://localhost:8111,auth-server-undertow-cross-dc-1.2-manual=http://localhost:8112</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-undertow-cross-dc-0.1" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-79</property>
+                <property name="route">auth-server-undertow-cross-dc-0.1</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">0</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.1",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+        <container qualifier="auth-server-undertow-cross-dc-0.2-manual" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-78</property>
+                <property name="route">auth-server-undertow-cross-dc-0.2</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">0</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.1",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-0.2",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+
+        <container qualifier="auth-server-undertow-cross-dc-1.1" mode="suite" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-69</property>
+                <property name="route">auth-server-undertow-cross-dc-1.1</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">1</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.1",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+        <container qualifier="auth-server-undertow-cross-dc-1.2-manual" mode="manual" >
+            <configuration>
+                <property name="enabled">${auth.server.undertow.crossdc}</property>
+                <property name="adapterImplClass">org.keycloak.testsuite.arquillian.undertow.KeycloakOnUndertow</property>
+                <property name="bindAddress">localhost</property>
+                <property name="bindHttpPort">${auth.server.http.port}</property>
+                <property name="bindHttpPortOffset">-68</property>
+                <property name="route">auth-server-undertow-cross-dc-1.2</property>
+                <property name="remoteMode">${undertow.remote}</property>
+                <property name="dataCenter">1</property>
+                <property name="keycloakConfigPropertyOverrides">{
+                    "keycloak.connectionsInfinispan.jgroupsUdpMcastAddr": "234.56.78.2",
+                    "keycloak.connectionsInfinispan.nodeName": "auth-server-undertow-cross-dc-1.2",
+                    "keycloak.connectionsInfinispan.clustered": "${keycloak.connectionsInfinispan.clustered:true}",
+                    "keycloak.connectionsInfinispan.remoteStoreServer": "${keycloak.connectionsInfinispan.remoteStoreServer:localhost}",
+                    "keycloak.connectionsInfinispan.remoteStorePort": "${keycloak.connectionsInfinispan.remoteStorePort:11222}",
+                    "keycloak.connectionsInfinispan.remoteStoreEnabled": "${keycloak.connectionsInfinispan.remoteStoreEnabled:true}",
+                    "keycloak.connectionsJpa.url": "${keycloak.connectionsJpa.url.crossdc:jdbc:h2:mem:test-dc-shared}",
+                    "keycloak.connectionsJpa.driver": "${keycloak.connectionsJpa.driver.crossdc:org.h2.Driver}",
+                    "keycloak.connectionsJpa.driverDialect": "${keycloak.connectionsJpa.driverDialect.crossdc:}"
+                }</property>
+            </configuration>
+        </container>
+    </group>
+
+
     <container qualifier="auth-server-balancer-wildfly" mode="suite" >
         <configuration>
             <property name="enabled">${auth.server.cluster}</property>

testsuite/integration-arquillian/tests/pom.xml

@@ -41,6 +41,7 @@
     <properties>
         <auth.server>undertow</auth.server>
         <auth.server.undertow>true</auth.server.undertow>
+        <auth.server.undertow.crossdc>true</auth.server.undertow.crossdc>
 
         <auth.server.container>auth-server-${auth.server}</auth.server.container>
         <auth.server.home>${containers.home}/${auth.server.container}</auth.server.home>
@@ -66,6 +67,16 @@
         <auth.server.profile/>
         <auth.server.feature/>
 
+        <cache.server>infinispan</cache.server>
+        <cache.server.container>cache-server-${cache.server}</cache.server.container>
+        <cache.server.home>${containers.home}/${cache.server.container}</cache.server.home>
+        <cache.server.port.offset>1010</cache.server.port.offset>
+        <cache.server.management.port>11000</cache.server.management.port>
+        <cache.server.console.output>true</cache.server.console.output>
+        <keycloak.connectionsInfinispan.remoteStoreServer>localhost</keycloak.connectionsInfinispan.remoteStoreServer>
+        <keycloak.connectionsInfinispan.remoteStorePort>12232</keycloak.connectionsInfinispan.remoteStorePort>
+        <keycloak.connectionsJpa.url.crossdc>jdbc:h2:mem:test-dc-shared</keycloak.connectionsJpa.url.crossdc>
+
         <adapter.test.props/>
         <migration.import.properties/>
         <examples.home>${project.build.directory}/examples</examples.home>
@@ -289,6 +300,7 @@
                 <auth.server>wildfly</auth.server>
                 <auth.server.jboss>true</auth.server.jboss>
                 <auth.server.undertow>false</auth.server.undertow>
+                <auth.server.undertow.crossdc>false</auth.server.undertow.crossdc>
                 <auth.server.config.property.value>standalone.xml</auth.server.config.property.value>
                 <auth.server.config.dir>${auth.server.home}/standalone/configuration</auth.server.config.dir>
                 <h2.version>1.3.173</h2.version>
@@ -307,6 +319,7 @@
                 <auth.server>eap</auth.server>
                 <auth.server.jboss>true</auth.server.jboss>
                 <auth.server.undertow>false</auth.server.undertow>
+                <auth.server.undertow.crossdc>false</auth.server.undertow.crossdc>
                 <auth.server.config.property.value>standalone.xml</auth.server.config.property.value>
                 <auth.server.config.dir>${auth.server.home}/standalone/configuration</auth.server.config.dir>
                 <h2.version>1.3.173</h2.version>
@@ -319,6 +332,110 @@
             </dependencies>
         </profile>
 
+        <profile>
+            <id>cache-server-infinispan</id>
+            <properties>
+                <cache.server>infinispan</cache.server>
+                <cache.server.jboss>true</cache.server.jboss>
+                <cache.server.config.dir>${cache.server.home}/standalone/configuration</cache.server.config.dir>                
+            </properties>
+            <dependencies>
+                <dependency>
+                    <groupId>org.wildfly.arquillian</groupId>
+                    <artifactId>wildfly-arquillian-container-managed</artifactId>
+                </dependency>
+<!--                <dependency>
+                    <groupId>org.infinispan.arquillian.container</groupId>
+                    <artifactId>infinispan-arquillian-impl</artifactId>
+                    <scope>test</scope>
+                    <exclusions>
+                        <exclusion>
+                            <groupId>org.infinispan</groupId>
+                            <artifactId>infinispan-core</artifactId>
+                        </exclusion>
+                    </exclusions>
+                </dependency>-->
+            </dependencies>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-enforcer-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <goals>
+                                    <goal>enforce</goal>
+                                </goals>
+                                <configuration>
+                                    <rules>
+                                        <!--requireActiveProfile 'auth-server-wildfly/eap' doesn't work unless the profiles are defined in all submodule poms
+                                        using requireProperty instead-->
+                                        <requireProperty>
+                                            <property>cache.server</property>
+                                            <regex>(infinispan)|(jdg)</regex>
+                                            <regexMessage>Profile "cache-server-infinispan" requires activation of profile "cache-server-infinispan" or "cache-server-jdg".</regexMessage>
+                                        </requireProperty>
+                                    </rules>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <artifactId>maven-antrun-plugin</artifactId>
+                    </plugin>
+                </plugins>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <artifactId>maven-dependency-plugin</artifactId>
+                            <executions>
+                                <execution>
+                                    <id>unpack-cache-server-infinispan</id>
+                                    <phase>generate-resources</phase>
+                                    <goals>
+                                        <goal>unpack</goal>
+                                    </goals>
+                                    <configuration>
+                                        <artifactItems>
+                                            <artifactItem>
+                                                <groupId>org.keycloak.testsuite</groupId>
+                                                <artifactId>integration-arquillian-servers-cache-server-infinispan</artifactId>
+                                                <version>${project.version}</version>
+                                                <type>zip</type>
+                                                <outputDirectory>${containers.home}</outputDirectory>
+                                            </artifactItem>
+                                        </artifactItems>
+                                        <overWriteIfNewer>true</overWriteIfNewer>
+                                    </configuration>
+                                </execution>
+                            </executions>
+                        </plugin>
+                        <plugin>
+                            <artifactId>maven-surefire-plugin</artifactId>
+                            <configuration>
+                                <systemPropertyVariables>
+                                    <auth.server.crossdc>true</auth.server.crossdc>
+                                    <auth.server.undertow.crossdc>${auth.server.undertow.crossdc}</auth.server.undertow.crossdc>
+
+                                    <cache.server>${cache.server}</cache.server>
+                                    <cache.server.port.offset>${cache.server.port.offset}</cache.server.port.offset>
+                                    <cache.server.container>${cache.server.container}</cache.server.container>
+                                    <cache.server.home>${cache.server.home}</cache.server.home>
+                                    <cache.server.console.output>${cache.server.console.output}</cache.server.console.output>
+                                    <cache.server.management.port>${cache.server.management.port}</cache.server.management.port>
+                                    
+                                    <keycloak.connectionsInfinispan.remoteStorePort>${keycloak.connectionsInfinispan.remoteStorePort}</keycloak.connectionsInfinispan.remoteStorePort>
+                                    <keycloak.connectionsInfinispan.remoteStoreServer>${keycloak.connectionsInfinispan.remoteStoreServer}</keycloak.connectionsInfinispan.remoteStoreServer>
+                                    
+                                    <keycloak.connectionsJpa.url.crossdc>${keycloak.connectionsJpa.url.crossdc}</keycloak.connectionsJpa.url.crossdc>
+                                </systemPropertyVariables>
+                            </configuration>
+                        </plugin>
+                    </plugins>
+                </pluginManagement>
+            </build>
+        </profile>
+
         <profile>
             <id>auth-server-profile</id>
             <activation>